def new_pipe_meta(artifact_path=None, *args):
from kfp.dsl import PipelineConf
def _set_artifact_path(task):
from kubernetes import client as k8s_client
task.add_env_variable(
k8s_client.V1EnvVar(name='MLRUN_ARTIFACT_PATH',
value=artifact_path))
return task
conf = PipelineConf()
if artifact_path:
conf.add_op_transformer(_set_artifact_path)
for op in args:
if op:
conf.add_op_transformer(op)
return conf
def _generate_pipeline_conf(self, pipeline: dict) -> PipelineConf:
"""
Returns a KFP pipeline configuration for this pipeline, which can be empty.
:param pipeline: pipeline dictionary
:type pipeline: dict
:return: https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.PipelineConf
:rtype: kfp.dsl import PipelineConf
"""
self.log.debug("Generating pipeline configuration ...")
pipeline_conf = PipelineConf()
#
# Gather input for container image pull secrets in support of private container image registries
# https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.PipelineConf.set_image_pull_secrets
#
image_namespace = self._get_metadata_configuration(
schemaspace=RuntimeImages.RUNTIME_IMAGES_SCHEMASPACE_ID)
# iterate through pipeline operations and create list of Kubernetes secret names
# that are associated with generic components
container_image_pull_secret_names = []
for operation in pipeline.operations.values():
if isinstance(operation, GenericOperation):
for image_instance in image_namespace:
if image_instance.metadata[
"image_name"] == operation.runtime_image:
if image_instance.metadata.get("pull_secret"):
container_image_pull_secret_names.append(
image_instance.metadata.get("pull_secret"))
break
if len(container_image_pull_secret_names) > 0:
# de-duplicate the pull secret name list, create Kubernetes resource
# references and add them to the pipeline configuration
container_image_pull_secrets = []
for secret_name in list(set(container_image_pull_secret_names)):
container_image_pull_secrets.append(
k8s_client.V1ObjectReference(name=secret_name))
pipeline_conf.set_image_pull_secrets(container_image_pull_secrets)
self.log.debug(
f"Added {len(container_image_pull_secrets)}"
" image pull secret(s) to the pipeline configuration.")
return pipeline_conf
def new_pipe_meta(artifact_path=None, ttl=None, *args):
from kfp.dsl import PipelineConf
def _set_artifact_path(task):
from kubernetes import client as k8s_client
task.add_env_variable(
k8s_client.V1EnvVar(name="MLRUN_ARTIFACT_PATH",
value=artifact_path))
return task
conf = PipelineConf()
ttl = ttl or int(config.kfp_ttl)
if ttl:
conf.set_ttl_seconds_after_finished(ttl)
if artifact_path:
conf.add_op_transformer(_set_artifact_path)
for op in args:
if op:
conf.add_op_transformer(op)
return conf
