def snapshot_pipeline_step(pipeline, step, nb_path, before=True):
"""Take a snapshot of a pipeline step with Rok."""
# Mark the start of the snapshotting procedure
log.info("%s Starting Rok snapshot procedure... (%s) %s", "-" * 10,
"before" if before else "after", "-" * 10)
log.info("Retrieving KFP run ID...")
run_uuid = podutils.get_run_uuid()
log.info("Retrieved KFP run ID: %s", run_uuid)
bucket = kfputils.get_experiment_from_run_id(run_uuid).name
obj = "{}-{}".format(pipeline, run_uuid)
commit_title = "Step: {} ({})".format(step, "start" if before else "end")
commit_message = "Autosnapshot {} step '{}' of pipeline run '{}'".format(
"before" if before else "after", step, run_uuid)
environment = json.dumps({
"KALE_PIPELINE_STEP": step,
"KALE_NOTEBOOK_PATH": nb_path,
"KALE_SNAPSHOT_FINAL": not before
})
metadata = json.dumps({
"environment": environment,
"kfp_runid": kfputils.format_kfp_run_id_uri(run_uuid),
"state": "initial" if before else "final"
})
params = {
"pod": podutils.get_pod_name(),
"metadata": metadata,
"default_container": "main",
"commit_title": commit_title,
"commit_message": commit_message
}
rok = get_client()
# Create the bucket in case it does not exist
create_rok_bucket(bucket)
log.info("Registering Rok version for '%s/%s'...", bucket, obj)
task_info = rok.version_register(bucket, obj, "pod", params, wait=True)
# FIXME: How do we retrieve the base URL of the ROK UI?
version = task_info["task"]["result"]["event"]["version"]
url_path = (
"/rok/buckets/%s/files/%s/versions/%s?ns=%s" %
(utils.encode_url_component(bucket), utils.encode_url_component(obj),
utils.encode_url_component(version),
utils.encode_url_component(podutils.get_namespace())))
log.info("Successfully registered Rok version '%s'", version)
log.info("Successfully created snapshot for step '%s'", step)
if before:
log.info("You can explore the state of the notebook at the beginning"
" of this step by spawning a new notebook from the following"
" Rok snapshot:")
log.info("%s", url_path)
reproduce_steps = ("To **explore the execution state** at the **%s** of"
" this step follow the instructions below:\n\n"
"1\\. View the [snapshot in the Rok UI](%s).\n\n"
"2\\. Copy the Rok URL.\n\n"
"3\\. Create a new Notebook Server by using this Rok"
" URL to autofill the form.")
if before:
md_source = (("# Rok autosnapshot\n"
"Rok has successfully created a snapshot for step `%s`."
"\n\n" + reproduce_steps) %
(step, "beginning", url_path))
else:
md_source = (("# Rok final autosnapshot\n"
"Rok has successfully created a snapshot **after** the"
" execution of step `%s`.\n\n" + reproduce_steps) %
(step, "end", url_path))
try:
metadataui = kfputils.get_current_uimetadata(default_if_not_exist=True)
except json.JSONDecodeError:
log.error("This step will not create a Rok markdown artifact.")
else:
metadataui["outputs"].append({
"storage": "inline",
"source": md_source,
"type": "markdown"
})
with open(kfputils.KFP_UI_METADATA_FILE_PATH, "w") as f:
json.dump(metadataui, f)
# Mark the end of the snapshotting procedure
log.info("%s Successfully ran Rok snapshot procedure (%s) %s", "-" * 10,
"before" if before else "after", "-" * 10)
return task_info
def snapshot_pipeline_step(pipeline, step, nb_path, before=True):
"""Take a snapshot of a pipeline step with Rok."""
# Mark the start of the snapshotting procedure
log.info("%s Starting Rok snapshot procedure... (%s) %s", "-" * 10,
"before" if before else "after", "-" * 10)
from rok_gw_client.client import RokClient
log.info("Retrieving KFP run ID...")
run_uuid = get_run_uuid()
log.info("Retrieved KFP run ID: %s", run_uuid)
bucket = kfputils.get_experiment_from_run_id(run_uuid).name
obj = "{}-{}".format(pipeline, run_uuid)
commit_title = "Step: {} ({})".format(step, "start" if before else "end")
commit_message = "Autosnapshot {} step '{}' of pipeline run '{}'".format(
"before" if before else "after", step, run_uuid)
environment = json.dumps({
"KALE_PIPELINE_STEP": step,
"KALE_NOTEBOOK_PATH": nb_path
})
metadata = json.dumps({"environment": environment, "kfp_runid": run_uuid})
params = {
"pod": get_pod_name(),
"metadata": metadata,
"default_container": "main",
"commit_title": commit_title,
"commit_message": commit_message
}
rok = RokClient()
# Create the bucket in case it does not exist
create_rok_bucket(bucket, client=rok)
log.info("Registering Rok version for '%s/%s'...", bucket, obj)
task_info = rok.version_register(bucket, obj, "pod", params, wait=True)
# FIXME: How do we retrieve the base URL of the ROK UI?
version = task_info["task"]["result"]["event"]["version"]
url_path = ("/rok/buckets/%s/files/%s/versions/%s" %
(encode_url_component(bucket), encode_url_component(obj),
encode_url_component(version)))
log.info("Successfully registered Rok version '%s'", version)
log.info("Successfully created snapshot for step '%s'", step)
if before:
log.info("You can explore the state of the notebook at the beginning"
" of this step by spawning a new notebook from the following"
" Rok snapshot:")
log.info("%s", url_path)
md_source = ("# Rok autosnapshot\n"
"Rok has successfully created a snapshot for step `%s`.\n\n"
"To **explore the execution state** at the beginning of "
"this step follow the instructions below:\n\n"
"1\\. View the [snapshot in the Rok UI](%s).\n\n"
"2\\. Copy the Rok URL.\n\n"
"3\\. Create a new Notebook Server by using this Rok URL to "
"autofill the form." % (step, url_path))
if before:
metadata = {
"outputs": [{
"storage": "inline",
"source": md_source,
"type": "markdown"
}]
}
with open("/mlpipeline-ui-metadata.json", "w") as f:
json.dump(metadata, f)
# Mark the end of the snapshotting procedure
log.info("%s Successfully ran Rok snapshot procedure (%s) %s", "-" * 10,
"before" if before else "after", "-" * 10)
return task_info