def _store_pipeline(yaml_file_content: AnyStr, name=None, description=None):
yaml_dict = yaml.load(yaml_file_content, Loader=yaml.FullLoader)
template_metadata = yaml_dict.get("metadata") or dict()
annotations = template_metadata.get("annotations", {})
pipeline_spec = json.loads(annotations.get("pipelines.kubeflow.org/pipeline_spec", "{}"))
name = name or template_metadata["name"]
description = pipeline_spec.get("description", "").strip()
namespace = pipeline_spec.get("namespace", "").strip()
pipeline_id = "-".join([generate_id(length=l) for l in [8, 4, 4, 4, 12]])
created_at = datetime.now()
parameters = [ApiParameter(name=p.get("name"), description=p.get("description"),
default=p.get("default"), value=p.get("value"))
for p in yaml_dict["spec"].get("params", {})]
api_pipeline = ApiPipeline(id=pipeline_id,
created_at=created_at,
name=name,
description=description,
parameters=parameters,
namespace=namespace)
uuid = store_data(api_pipeline)
api_pipeline.id = uuid
store_file(bucket_name="mlpipeline", prefix=f"pipelines/",
file_name=f"{pipeline_id}", file_content=yaml_file_content)
enable_anonymous_read_access(bucket_name="mlpipeline", prefix="pipelines/*")
return api_pipeline
def _upload_model_yaml(yaml_file_content: AnyStr, name=None, existing_id=None):
model_def = yaml.load(yaml_file_content, Loader=yaml.FullLoader)
api_model = ApiModel(
id=existing_id or model_def.get("model_identifier")
or generate_id(name=name or model_def["name"]),
created_at=datetime.now(),
name=name or model_def["name"],
description=model_def["description"].strip(),
domain=model_def.get("domain") or "",
labels=model_def.get("labels") or dict(),
framework=model_def["framework"],
filter_categories=model_def.get("filter_categories") or dict(),
trainable=model_def.get("train", {}).get("trainable") or False,
trainable_tested_platforms=model_def.get(
"train", {}).get("tested_platforms") or [],
trainable_credentials_required=model_def.get(
"train", {}).get("credentials_required") or False,
trainable_parameters=model_def.get("train", {}).get("input_params")
or [],
servable=model_def.get("serve", {}).get("servable") or False,
servable_tested_platforms=model_def.get(
"serve", {}).get("tested_platforms") or [],
servable_credentials_required=model_def.get(
"serve", {}).get("credentials_required") or False,
servable_parameters=model_def.get("serve", {}).get("input_params")
or [])
# convert comma-separate strings to lists
if type(api_model.trainable_tested_platforms) == str:
api_model.trainable_tested_platforms = api_model.trainable_tested_platforms.replace(
", ", ",").split(",")
if type(api_model.servable_tested_platforms) == str:
api_model.servable_tested_platforms = api_model.servable_tested_platforms.replace(
", ", ",").split(",")
uuid = store_data(api_model)
api_model.id = uuid
store_file(bucket_name="mlpipeline",
prefix=f"models/{api_model.id}/",
file_name="template.yaml",
file_content=yaml_file_content,
content_type="text/yaml")
enable_anonymous_read_access(bucket_name="mlpipeline", prefix="models/*")
return api_model, 201
def _upload_component_yaml(yaml_file_content: AnyStr,
name=None,
existing_id=None):
yaml_dict = yaml.load(yaml_file_content, Loader=yaml.FullLoader)
template_metadata = yaml_dict.get("metadata") or dict()
component_id = existing_id or generate_id(name=name or yaml_dict["name"])
created_at = datetime.now()
name = name or yaml_dict["name"]
description = (yaml_dict.get("description") or name).strip()[:255]
filter_categories = yaml_dict.get("filter_categories") or dict()
metadata = ApiMetadata(annotations=template_metadata.get("annotations"),
labels=template_metadata.get("labels"),
tags=template_metadata.get("tags"))
parameters = [
ApiParameter(name=p.get("name"),
description=p.get("description"),
default=p.get("default"),
value=p.get("value"))
for p in yaml_dict.get("inputs", [])
]
api_component = ApiComponent(id=component_id,
created_at=created_at,
name=name,
description=description,
metadata=metadata,
parameters=parameters,
filter_categories=filter_categories)
uuid = store_data(api_component)
api_component.id = uuid
store_file(bucket_name="mlpipeline",
prefix=f"components/{component_id}/",
file_name="template.yaml",
file_content=yaml_file_content,
content_type="text/yaml")
enable_anonymous_read_access(bucket_name="mlpipeline",
prefix="components/*")
return api_component, 201
def run_component(id, parameters, run_name=None): # noqa: E501
"""run_component
:param id:
:type id: str
:param parameters:
:type parameters: List[ApiParameter]
:param run_name: name to identify the run on the Kubeflow Pipelines UI, defaults to component name
:type run_name: str
:rtype: ApiRunCodeResponse
"""
if KFP_HOST == "UNAVAILABLE":
return f"Kubeflow Pipeline host is 'UNAVAILABLE'", 503
if connexion.request.is_json:
parameters = [
ApiParameter.from_dict(d) for d in connexion.request.get_json()
] # noqa: E501
parameter_dict = {
p.name: p.value
for p in parameters if p.value and p.value.strip() != ""
}
api_component, status_code = get_component(id)
if status_code > 200:
return f"Component with id '{id}' does not exist", 404
parameter_errors, status_code = validate_parameters(
api_component.parameters, parameter_dict)
if parameter_errors:
return parameter_errors, status_code
api_template, _ = get_component_template(id)
enable_anonymous_read_access(bucket_name="mlpipeline",
prefix="components/*")
try:
run_id = run_component_in_experiment(api_component, api_template.url,
parameter_dict, run_name)
return ApiRunCodeResponse(run_url=f"/runs/details/{run_id}"), 200
except Exception as e:
return f"Error while trying to run component {id}: {e}", 500
def run_custom_pipeline(run_custom_pipeline_payload, run_name=None): # noqa: E501
"""run_custom_pipeline
Run a complex pipeline defined by a directed acyclic graph (DAG)
:param run_custom_pipeline_payload: A custom pipeline defined by a directed acyclic graph (DAG) and input parameters
:type run_custom_pipeline_payload: dict | bytes
:param run_name: Name to identify the run on the Kubeflow Pipelines UI
:type run_name: str
:rtype: ApiRunCodeResponse
"""
if connexion.request.is_json:
run_custom_pipeline_payload = ApiPipelineCustomRunPayload.from_dict(connexion.request.get_json()) # noqa: E501
run_parameters = run_custom_pipeline_payload.run_parameters or {}
custom_pipeline = run_custom_pipeline_payload.custom_pipeline
# ensure unique task names
task_names = [t.name for t in custom_pipeline.dag.tasks]
duplicate_task_names = [name for name, count in Counter(task_names).items() if count > 1]
assert not duplicate_task_names, f"duplicate task names: {duplicate_task_names}"
# validate pipeline dependencies
pipeline_tasks_by_name: typing.Dict[str, ApiPipelineTask] = {t.name: t for t in custom_pipeline.dag.tasks}
for t in pipeline_tasks_by_name.values():
for required_task_name in t.dependencies or []:
assert required_task_name in pipeline_tasks_by_name, \
f"missing task '{required_task_name}', as dependency for task '{t.name}'"
# validate input parameters
missing_run_parameters = {p.name for p in custom_pipeline.inputs.parameters
if p.default is None and p.value is None} - run_parameters.keys()
assert not missing_run_parameters, f"missing parameters to run pipeline: {missing_run_parameters}"
# make sure we enable anonymous read access to pipeline task components
for artifact_type in set([t.artifact_type for t in pipeline_tasks_by_name.values()]):
enable_anonymous_read_access(bucket_name="mlpipeline", prefix=f"{artifact_type}s/*")
try:
run_id = run_custom_pipeline_in_experiment(custom_pipeline, run_name, run_parameters)
return ApiRunCodeResponse(run_url=f"/runs/details/{run_id}"), 200
except Exception as e:
# TODO: remove traceback?
import traceback
print(traceback.format_exc())
return f"Error while trying to run custom pipeline '{run_name}': {e}", 500
def _upload_notebook_yaml(yaml_file_content: AnyStr,
name=None,
access_token=None,
existing_id=None):
yaml_dict = yaml.load(yaml_file_content, Loader=yaml.FullLoader)
template_metadata = yaml_dict.get("metadata") or dict()
notebook_id = existing_id or generate_id(name=name or yaml_dict["name"])
created_at = datetime.now()
name = name or yaml_dict["name"]
description = yaml_dict["description"].strip()
url = yaml_dict["implementation"]["github"]["source"]
requirements = yaml_dict["implementation"]["github"].get("requirements")
metadata = ApiMetadata(annotations=template_metadata.get("annotations"),
labels=template_metadata.get("labels"),
tags=template_metadata.get("tags"))
notebook_content = _download_notebook(
url, enterprise_github_api_token=access_token)
# parameters = _extract_notebook_parameters(notebook_content)
# TODO: not using Papermill any longer, notebook parameters no longer valid?
# kfp-notebook has inputs and outputs ?
parameters = dict()
api_notebook = ApiNotebook(id=notebook_id,
created_at=created_at,
name=name,
description=description,
url=url,
metadata=metadata,
parameters=parameters)
uuid = store_data(api_notebook)
api_notebook.id = uuid
store_file(bucket_name="mlpipeline",
prefix=f"notebooks/{notebook_id}/",
file_name="template.yaml",
file_content=yaml_file_content)
s3_url = store_file(bucket_name="mlpipeline",
prefix=f"notebooks/{notebook_id}/",
file_name=url.split("/")[-1].split("?")[0],
file_content=json.dumps(notebook_content).encode())
if requirements:
if _is_url(requirements):
requirements_url = requirements
requirements_txt = download_file_content_from_url(
requirements_url).decode()
else:
requirements_txt = "\n".join(requirements.split(","))
# TODO: remove this after fixing the Elyra-AI/KFP-Notebook runner so that
# Elyra should install its own requirements in addition to the provided requirements
requirements_elyra_url = "https://github.com/elyra-ai/kfp-notebook/blob/master/etc/requirements-elyra.txt"
requirements_elyra_txt = download_file_content_from_url(
requirements_elyra_url).decode()
requirements_elyra = "\n".join([
line for line in requirements_elyra_txt.split("\n")
if not line.startswith("#")
])
requirements_all = f"# Required packages for {api_notebook.name}:\n" \
f"{requirements_txt}\n" \
f"# Requirements from {requirements_elyra_url}:\n" \
f"{requirements_elyra}"
store_file(bucket_name="mlpipeline",
prefix=f"notebooks/{notebook_id}/",
file_name="requirements.txt",
file_content=requirements_all.encode())
# if the url included an access token, replace the original url with the s3 url
if "?token=" in url or "github.ibm.com" in url:
api_notebook.url = s3_url
update_multiple(ApiNotebook, [notebook_id], "url", s3_url)
enable_anonymous_read_access(bucket_name="mlpipeline",
prefix="notebooks/*")
return api_notebook, 201
def _upload_dataset_yaml(yaml_file_content: AnyStr,
name=None,
existing_id=None):
yaml_dict = yaml.load(yaml_file_content, Loader=yaml.FullLoader)
name = name or yaml_dict["name"]
description = yaml_dict["description"]
dataset_id = existing_id or generate_id(name=yaml_dict.get("id", name))
created_at = datetime.now()
# if yaml_dict.get("id") != dataset_id:
# raise ValueError(f"Dataset.id contains non k8s character: {yaml_dict.get('id')}")
# TODO: re-evaluate if we should use dataset update time as our MLX "created_at" time
if "updated" in yaml_dict:
created_at = datetime.strptime(str(yaml_dict["updated"]), "%Y-%m-%d")
elif "created" in yaml_dict:
created_at = datetime.strptime(str(yaml_dict["created"]), "%Y-%m-%d")
license_name = yaml_dict["license"]["name"]
domain = yaml_dict["domain"]
format_type = yaml_dict["format"][0]["type"]
size = yaml_dict["content"][0].get("size")
version = yaml_dict["version"]
# # extract number of records and convert thousand separators based on Locale
# num_records_str = yaml_dict["statistics"]["number_of_records"]
# num_records_number_str = num_records_str.split()[0]. \
# replace("~", ""). \
# replace("+", ""). \
# replace("k", "000"). \
# replace(",", "") # assumes thousand separators in locale.en_US.UTF-8
# # locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') # setting locale does not work reliably in Docker
# # number_of_records = locale.atoi(num_records_number_str)
# number_of_records = int(num_records_number_str)
number_of_records = yaml_dict["content"][0].get("records", 0)
related_assets = [
a["application"].get("asset_id")
for a in yaml_dict.get("related_assets", [])
if "MLX" in a.get("application", {}).get("name", "")
and "asset_id" in a.get("application", {})
]
template_metadata = yaml_dict.get("metadata") or dict()
metadata = ApiMetadata(annotations=template_metadata.get("annotations"),
labels=template_metadata.get("labels"),
tags=template_metadata.get("tags")
or yaml_dict.get("seo_tags"))
# TODO: add "version" to ApiDataset
api_dataset = ApiDataset(id=dataset_id,
created_at=created_at,
name=name,
description=description,
domain=domain,
format=format_type,
size=size,
number_of_records=number_of_records,
license=license_name,
metadata=metadata,
related_assets=related_assets)
uuid = store_data(api_dataset)
api_dataset.id = uuid
store_file(bucket_name="mlpipeline",
prefix=f"datasets/{api_dataset.id}/",
file_name="template.yaml",
file_content=yaml_file_content)
enable_anonymous_read_access(bucket_name="mlpipeline", prefix="datasets/*")
return api_dataset, 201
def run_notebook(id, run_name=None, parameters: dict = None): # noqa: E501
"""run_notebook
:param id:
:type id: str
:param run_name: name to identify the run on the Kubeflow Pipelines UI, defaults to notebook name
:type run_name: str
:param parameters: optional run parameters, may be required based on pipeline definition
:type parameters: dict
:rtype: ApiRunCodeResponse
"""
if KFP_HOST == "UNAVAILABLE":
return f"Kubeflow Pipeline host is 'UNAVAILABLE'", 503
if not parameters and connexion.request.is_json:
parameter_dict = dict(connexion.request.get_json()) # noqa: E501
else:
parameter_dict = parameters
api_notebook, status_code = get_notebook(id)
if status_code > 200:
return f"Notebook with id '{id}' does not exist", 404
# # TODO: Elyra kfp-notebook currently does not pass parameters on to papermill
# if parameters:
# raise ApiError("The 'elyra-ai/kfp-notebook' executor does not support parameters", 422)
# parameter_errors, status_code = validate_parameters(api_notebook.parameters, parameter_dict)
#
# if parameter_errors:
# return parameter_errors, status_code
# Elyra pulls the requirements.txt from Minio, requiring anonymous read access
enable_anonymous_read_access(bucket_name="mlpipeline",
prefix="notebooks/*")
try:
run_id = run_notebook_in_experiment(notebook=api_notebook,
parameters=parameter_dict,
run_name=run_name)
# expected output notebook based on:
# https://github.com/elyra-ai/kfp-notebook/blob/c8f1298/etc/docker-scripts/bootstrapper.py#L188-L190
notebook_url = get_object_url(bucket_name="mlpipeline",
prefix=f"notebooks/{api_notebook.id}/",
file_extensions=[".ipynb"])
# TODO: create a "sandboxed" notebook in a subfolder since Elyra overwrites
# the original notebook instead of creating an "-output.ipynb" file:
# https://github.com/elyra-ai/kfp-notebook/blob/c8f1298/etc/docker-scripts/bootstrapper.py#L205
notebook_output_url = notebook_url.replace(".ipynb", "-output.ipynb")
# instead return link to the generated output .html for the time being
notebook_output_html = notebook_url.replace(".ipynb", ".html")
return ApiRunCodeResponse(
run_url=f"/runs/details/{run_id}",
run_output_location=notebook_output_html), 200
except Exception as e:
return f"Error while trying to run notebook {id}: {e}", 500
