def test_workflow_without_decorator(pipeline_mod, params_dict):
"""Test compiling a workflow and appending pipeline params."""
try:
pipeline_params = []
for param in params_dict.get('paramsList', []):
pipeline_params.append(getattr(pipeline_mod, param))
compiled_workflow = compiler.TektonCompiler()._create_workflow(
getattr(pipeline_mod, params_dict['function']),
params_dict.get('name',
None), params_dict.get('description', None),
pipeline_params if pipeline_params else None,
params_dict.get('conf', None))
return True
except:
return False
def _test_pipeline_workflow(self,
pipeline_function,
pipeline_yaml,
normalize_compiler_output_function=None):
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml)
temp_dir = tempfile.mkdtemp()
compiled_yaml_file = os.path.join(temp_dir, 'workflow.yaml')
try:
compiler.TektonCompiler().compile(pipeline_function,
compiled_yaml_file)
with open(compiled_yaml_file, 'r') as f:
f = normalize_compiler_output_function(
f.read()) if normalize_compiler_output_function else f
compiled = yaml.safe_load(f)
self._verify_compiled_workflow(golden_yaml_file, compiled)
finally:
shutil.rmtree(temp_dir)
def _test_workflow_without_decorator(self, pipeline_yaml, params_dict):
"""
Test compiling a workflow and appending pipeline params.
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml)
temp_dir = tempfile.mkdtemp()
try:
kfp_tekton_compiler = compiler.TektonCompiler()
kfp_tekton_compiler.generate_pipelinerun = True
compiled_workflow = kfp_tekton_compiler._create_workflow(
params_dict['function'], params_dict.get('name', None),
params_dict.get('description', None),
params_dict.get('paramsList', None),
params_dict.get('conf', None))
self._verify_compiled_workflow(golden_yaml_file, compiled_workflow)
finally:
shutil.rmtree(temp_dir)
def _test_nested_workflow(self,
pipeline_yaml,
pipeline_list,
normalize_compiler_output_function=None):
"""
Test compiling a simple workflow, and a bigger one composed from the simple one.
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml)
temp_dir = tempfile.mkdtemp()
compiled_yaml_file = os.path.join(temp_dir, 'nested' + str(len(pipeline_list) - 1) + '.yaml')
try:
for index, pipeline in enumerate(pipeline_list):
package_path = os.path.join(temp_dir, 'nested' + str(index) + '.yaml')
compiler.TektonCompiler().compile(pipeline,
package_path)
with open(compiled_yaml_file, 'r') as f:
f = normalize_compiler_output_function(
f.read()) if normalize_compiler_output_function else f
compiled = yaml.safe_load(f)
self._verify_compiled_workflow(golden_yaml_file, compiled)
finally:
shutil.rmtree(temp_dir)
def test_sequential_workflow(self):
"""
Test compiling a sequential workflow.
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_yaml_file = os.path.join(test_data_dir, 'sequential.yaml')
temp_dir = tempfile.mkdtemp()
compiled_yaml_file = os.path.join(temp_dir, 'workflow.yaml')
try:
from .testdata.sequential import sequential_pipeline
compiler.TektonCompiler().compile(sequential_pipeline, compiled_yaml_file)
with open(compiled_yaml_file, 'r') as f:
compiled = list(yaml.safe_load_all(f))
if GENERATE_GOLDEN_YAML:
with open(golden_yaml_file, 'w') as f:
yaml.dump_all(compiled, f, default_flow_style=False)
else:
with open(golden_yaml_file, 'r') as f:
golden = list(yaml.safe_load_all(f))
self.maxDiff = None
self.assertEqual(golden, compiled)
finally:
shutil.rmtree(temp_dir)
def process(self, pipeline):
"""Runs a pipeline on Kubeflow Pipelines
Each time a pipeline is processed, a new version
is uploaded and run under the same experiment name.
"""
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata['api_endpoint']
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_bucket = runtime_configuration.metadata['cos_bucket']
user_namespace = runtime_configuration.metadata.get('user_namespace')
# TODO: try to encapsulate the info below
api_username = runtime_configuration.metadata.get('api_username')
api_password = runtime_configuration.metadata.get('api_password')
engine = runtime_configuration.metadata.get('engine')
pipeline_name = pipeline.name
try:
# Connect to the Kubeflow server, determine whether it is secured,
# and if it is try to authenticate with the user-provided credentials
# (if any were defined in the runtime configuration)
endpoint = api_endpoint.replace('/pipeline', '')
auth_info = \
KfpPipelineProcessor._get_user_auth_session_cookie(endpoint,
api_username,
api_password)
self.log.debug(f"Kubeflow authentication info: {auth_info}")
if auth_info['endpoint_secured'] and \
auth_info['authservice_session_cookie'] is None:
# Kubeflow is secured but our attempt to authenticate did
# not yield the expected results. Log the collected authentication
# information and abort processing.
self.log.warning(f"Kubeflow authentication info: {auth_info}")
raise RuntimeError(f"Error connecting to Kubeflow at '{endpoint}'"
f": Authentication request failed. Check the "
f"Kubeflow Pipelines credentials in runtime "
f"configuration '{pipeline.runtime_config}'.")
# Create a KFP client
if 'Tekton' == engine:
client = TektonClient(host=api_endpoint,
cookies=auth_info['authservice_session_cookie'])
else:
client = ArgoClient(host=api_endpoint,
cookies=auth_info['authservice_session_cookie'])
# Determine whether a pipeline with the provided
# name already exists
pipeline_id = client.get_pipeline_id(pipeline_name)
if pipeline_id is None:
# The KFP default version name is the pipeline
# name
pipeline_version_name = pipeline_name
else:
# Append timestamp to generate unique version name
pipeline_version_name = f'{pipeline_name}-{timestamp}'
# Establish a 1:1 relationship with an experiment
# work around https://github.com/kubeflow/pipelines/issues/5172
experiment_name = pipeline_name.lower()
# Unique identifier for the pipeline run
job_name = f'{pipeline_name}-{timestamp}'
# Unique location on COS where the pipeline run artifacts
# will be stored
cos_directory = f'{pipeline_name}-{timestamp}'
except MaxRetryError as ex:
raise RuntimeError('Error connecting to pipeline server {}'.format(api_endpoint)) from ex
except LocationValueError as lve:
if api_username:
raise ValueError("Failure occurred uploading pipeline, check your credentials") from lve
else:
raise lve
# Verify that user-entered namespace is valid
try:
client.list_experiments(namespace=user_namespace,
page_size=0)
except ApiException as ae:
error_msg = f"{ae.reason} ({ae.status})"
if ae.body:
error_body = json.loads(ae.body)
error_msg += f": {error_body['error']}"
if error_msg[-1] not in ['.', '?', '!']:
error_msg += '.'
namespace = "namespace" if not user_namespace else f"namespace {user_namespace}"
self.log.error(f"Error validating {namespace}: {error_msg}")
raise RuntimeError(f"Error validating {namespace}: {error_msg} " +
"Please validate your runtime configuration details and retry.") from ae
self.log_pipeline_info(pipeline_name, "submitting pipeline")
with tempfile.TemporaryDirectory() as temp_dir:
pipeline_path = os.path.join(temp_dir, f'{pipeline_name}.tar.gz')
self.log.debug("Creating temp directory %s", temp_dir)
# Compile the new pipeline
try:
pipeline_function = lambda: self._cc_pipeline(pipeline, # nopep8 E731
pipeline_name=pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
cos_directory=cos_directory)
if 'Tekton' == engine:
kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, pipeline_path)
else:
kfp_argo_compiler.Compiler().compile(pipeline_function, pipeline_path)
except Exception as ex:
if ex.__cause__:
raise RuntimeError(str(ex)) from ex
raise RuntimeError('Error pre-processing pipeline {} for engine {} at {}'.
format(pipeline_name, engine, pipeline_path), str(ex)) from ex
self.log.debug("Kubeflow Pipeline was created in %s", pipeline_path)
# Upload the compiled pipeline, create an experiment and run
try:
description = f"Created with Elyra {__version__} pipeline editor using '{pipeline.source}'."
t0 = time.time()
if pipeline_id is None:
# Upload new pipeline. The call returns
# a unique pipeline id.
kfp_pipeline = \
client.upload_pipeline(pipeline_path,
pipeline_name,
description)
pipeline_id = kfp_pipeline.id
version_id = None
else:
# Upload a pipeline version. The call returns
# a unique version id.
kfp_pipeline = \
client.upload_pipeline_version(pipeline_path,
pipeline_version_name,
pipeline_id=pipeline_id)
version_id = kfp_pipeline.id
self.log_pipeline_info(pipeline_name, 'pipeline uploaded', duration=(time.time() - t0))
except MaxRetryError as ex:
raise RuntimeError('Error connecting to pipeline server {}'.format(api_endpoint)) from ex
except LocationValueError as lve:
if api_username:
raise ValueError("Failure occurred uploading pipeline, check your credentials") from lve
else:
raise lve
# Create a new experiment. If it already exists this is
# a no-op.
experiment = client.create_experiment(name=experiment_name,
namespace=user_namespace)
self.log_pipeline_info(pipeline_name,
f'Created experiment {experiment_name}',
duration=(time.time() - t0_all))
# Run the pipeline (or specified pipeline version)
run = client.run_pipeline(experiment_id=experiment.id,
job_name=job_name,
pipeline_id=pipeline_id,
version_id=version_id)
self.log_pipeline_info(pipeline_name,
f"pipeline submitted: {api_endpoint}/#/runs/details/{run.id}",
duration=(time.time() - t0_all))
return KfpPipelineProcessorResponse(
run_url=f'{api_endpoint}/#/runs/details/{run.id}',
object_storage_url=f'{cos_endpoint}',
object_storage_path=f'/{cos_bucket}/{cos_directory}',
)
return None
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite):
if pipeline_export_format not in ["yaml", "py"]:
raise ValueError("Pipeline export format {} not recognized.".format(pipeline_export_format))
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = pipeline.name
pipeline_version_name = f'{pipeline_name}-{timestamp}'
# work around https://github.com/kubeflow/pipelines/issues/5172
experiment_name = pipeline_name.lower()
# Unique identifier for the pipeline run
job_name = f'{pipeline_name}-{timestamp}'
# Unique location on COS where the pipeline run artifacts
# will be stored
cos_directory = f'{pipeline_name}-{timestamp}'
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path)
runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata['api_endpoint']
namespace = runtime_configuration.metadata.get('user_namespace')
engine = runtime_configuration.metadata.get('engine')
cos_secret = runtime_configuration.metadata.get('cos_secret')
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path + " already exists.")
self.log_pipeline_info(pipeline_name, f"exporting pipeline as a .{pipeline_export_format} file")
if pipeline_export_format != "py":
# Export pipeline as static configuration file (YAML formatted)
try:
# Exported pipeline is not associated with an experiment
# or a version. The association is established when the
# pipeline is imported into KFP by the user.
pipeline_function = lambda: self._cc_pipeline(pipeline,
pipeline_name,
cos_directory=cos_directory) # nopep8
if 'Tekton' == engine:
self.log.info("Compiling pipeline for Tekton engine")
kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, absolute_pipeline_export_path)
else:
self.log.info("Compiling pipeline for Argo engine")
kfp_argo_compiler.Compiler().compile(pipeline_function, absolute_pipeline_export_path)
except Exception as ex:
if ex.__cause__:
raise RuntimeError(str(ex)) from ex
raise RuntimeError('Error pre-processing pipeline {} for export at {}'.
format(pipeline_name, absolute_pipeline_export_path), str(ex)) from ex
else:
# Export pipeline as Python DSL
# Load template from installed elyra package
loader = PackageLoader('elyra', 'templates/kfp')
template_env = Environment(loader=loader, trim_blocks=True)
template_env.filters['to_basename'] = lambda path: os.path.basename(path)
template = template_env.get_template('kfp_template.jinja2')
defined_pipeline = self._cc_pipeline(pipeline,
pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
cos_directory=cos_directory,
export=True)
description = f'Created with Elyra {__version__} pipeline editor using {pipeline.source}.'
for key, operation in defined_pipeline.items():
self.log.debug("component :\n "
"container op name : %s \n "
"inputs : %s \n "
"outputs : %s \n ",
operation.name,
operation.inputs,
operation.outputs)
# The exported pipeline is by default associated with
# an experiment.
# The user can manually customize the generated code
# and change the associations as desired.
python_output = template.render(operations_list=defined_pipeline,
pipeline_name=pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
run_name=job_name,
engine=engine,
cos_secret=cos_secret,
namespace=namespace,
api_endpoint=api_endpoint,
pipeline_description=description,
writable_container_dir=self.WCD)
# Write to Python file and fix formatting
with open(absolute_pipeline_export_path, "w") as fh:
autopep_output = autopep8.fix_code(python_output)
output_to_file = format_str(autopep_output, mode=FileMode())
fh.write(output_to_file)
self.log_pipeline_info(pipeline_name, "pipeline rendered", duration=(time.time() - t0_all))
self.log_pipeline_info(pipeline_name,
f"pipeline exported: {pipeline_export_path}",
duration=(time.time() - t0_all))
return pipeline_export_path # Return the input value, not its absolute form
def process(self, pipeline):
"""
Runs a pipeline on Kubeflow Pipelines
Each time a pipeline is processed, a new version
is uploaded and run under the same experiment name.
"""
timestamp = datetime.now().strftime("%m%d%H%M%S")
################
# Runtime Configs
################
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID, name=pipeline.runtime_config
)
# unpack Kubeflow Pipelines configs
api_endpoint = runtime_configuration.metadata["api_endpoint"].rstrip("/")
api_username = runtime_configuration.metadata.get("api_username")
api_password = runtime_configuration.metadata.get("api_password")
user_namespace = runtime_configuration.metadata.get("user_namespace")
engine = runtime_configuration.metadata.get("engine")
if engine == "Tekton" and not TektonClient:
raise ValueError(
"Python package `kfp-tekton` is not installed. "
"Please install using `elyra[kfp-tekton]` to use Tekton engine."
)
# unpack Cloud Object Storage configs
cos_endpoint = runtime_configuration.metadata["cos_endpoint"]
cos_bucket = runtime_configuration.metadata["cos_bucket"]
# Determine which provider to use to authenticate with Kubeflow
auth_type = runtime_configuration.metadata.get("auth_type")
try:
auth_info = KFPAuthenticator().authenticate(
api_endpoint,
auth_type_str=auth_type,
runtime_config_name=pipeline.runtime_config,
auth_parm_1=api_username,
auth_parm_2=api_password,
)
self.log.debug(f"Authenticator returned {auth_info}")
except AuthenticationError as ae:
if ae.get_request_history() is not None:
self.log.info("An authentication error was raised. Diagnostic information follows.")
self.log.info(ae.request_history_to_string())
raise RuntimeError(f"Kubeflow authentication failed: {ae}")
#############
# Create Kubeflow Client
#############
try:
if engine == "Tekton":
client = TektonClient(
host=api_endpoint,
cookies=auth_info.get("cookies", None),
credentials=auth_info.get("credentials", None),
existing_token=auth_info.get("existing_token", None),
namespace=user_namespace,
)
else:
client = ArgoClient(
host=api_endpoint,
cookies=auth_info.get("cookies", None),
credentials=auth_info.get("credentials", None),
existing_token=auth_info.get("existing_token", None),
namespace=user_namespace,
)
except Exception as ex:
# a common cause of these errors is forgetting to include `/pipeline` or including it with an 's'
api_endpoint_obj = urlsplit(api_endpoint)
if api_endpoint_obj.path != "/pipeline":
api_endpoint_tip = api_endpoint_obj._replace(path="/pipeline").geturl()
tip_string = (
f" - [TIP: did you mean to set '{api_endpoint_tip}' as the endpoint, "
f"take care not to include 's' at end]"
)
else:
tip_string = ""
raise RuntimeError(
f"Failed to initialize `kfp.Client()` against: '{api_endpoint}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
f"{tip_string}"
) from ex
#############
# Verify Namespace
#############
try:
client.list_experiments(namespace=user_namespace, page_size=1)
except Exception as ex:
if user_namespace:
tip_string = f"[TIP: ensure namespace '{user_namespace}' is correct]"
else:
tip_string = "[TIP: you probably need to set a namespace]"
raise RuntimeError(
f"Failed to `kfp.Client().list_experiments()` against: '{api_endpoint}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}' - "
f"{tip_string}"
) from ex
#############
# Pipeline Metadata
#############
# generate a pipeline name
pipeline_name = pipeline.name
# generate a pipeline description
pipeline_description = pipeline.description
if pipeline_description is None:
pipeline_description = f"Created with Elyra {__version__} pipeline editor using `{pipeline.source}`."
#############
# Submit & Run the Pipeline
#############
self.log_pipeline_info(pipeline_name, "submitting pipeline")
with tempfile.TemporaryDirectory() as temp_dir:
self.log.debug(f"Created temporary directory at: {temp_dir}")
pipeline_path = os.path.join(temp_dir, f"{pipeline_name}.tar.gz")
#############
# Get Pipeline ID
#############
try:
# get the kubeflow pipeline id (returns None if not found, otherwise the ID of the pipeline)
pipeline_id = client.get_pipeline_id(pipeline_name)
# calculate what "pipeline version" name to use
if pipeline_id is None:
# the first "pipeline version" name must be the pipeline name
pipeline_version_name = pipeline_name
else:
# generate a unique name for a new "pipeline version" by appending the current timestamp
pipeline_version_name = f"{pipeline_name}-{timestamp}"
except Exception as ex:
raise RuntimeError(
f"Failed to get ID of Kubeflow pipeline: '{pipeline_name}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
) from ex
#############
# Compile the Pipeline
#############
try:
t0 = time.time()
# generate a name for the experiment (lowercase because experiments are case intensive)
experiment_name = pipeline_name.lower()
# unique location on COS where the pipeline run artifacts will be stored
cos_directory = f"{pipeline_name}-{timestamp}"
pipeline_function = lambda: self._cc_pipeline( # nopep8 E731
pipeline,
pipeline_name=pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
cos_directory=cos_directory,
)
# collect pipeline configuration information
pipeline_conf = self._generate_pipeline_conf(pipeline)
# compile the pipeline
if engine == "Tekton":
kfp_tekton_compiler.TektonCompiler().compile(
pipeline_function, pipeline_path, pipeline_conf=pipeline_conf
)
else:
kfp_argo_compiler.Compiler().compile(pipeline_function, pipeline_path, pipeline_conf=pipeline_conf)
except RuntimeError:
raise
except Exception as ex:
raise RuntimeError(
f"Failed to compile pipeline '{pipeline_name}' with engine '{engine}' to: '{pipeline_path}'"
) from ex
self.log_pipeline_info(pipeline_name, "pipeline compiled", duration=time.time() - t0)
#############
# Upload Pipeline Version
#############
try:
t0 = time.time()
# CASE 1: pipeline needs to be created
if pipeline_id is None:
# create new pipeline (and initial "pipeline version")
kfp_pipeline = client.upload_pipeline(
pipeline_package_path=pipeline_path,
pipeline_name=pipeline_name,
description=pipeline_description,
)
# extract the ID of the pipeline we created
pipeline_id = kfp_pipeline.id
# the initial "pipeline version" has the same id as the pipeline itself
version_id = pipeline_id
# CASE 2: pipeline already exists
else:
# upload the "pipeline version"
kfp_pipeline = client.upload_pipeline_version(
pipeline_package_path=pipeline_path,
pipeline_version_name=pipeline_version_name,
pipeline_id=pipeline_id,
)
# extract the id of the "pipeline version" that was created
version_id = kfp_pipeline.id
except Exception as ex:
# a common cause of these errors is forgetting to include `/pipeline` or including it with an 's'
api_endpoint_obj = urlsplit(api_endpoint)
if api_endpoint_obj.path != "/pipeline":
api_endpoint_tip = api_endpoint_obj._replace(path="/pipeline").geturl()
tip_string = (
f" - [TIP: did you mean to set '{api_endpoint_tip}' as the endpoint, "
f"take care not to include 's' at end]"
)
else:
tip_string = ""
raise RuntimeError(
f"Failed to upload Kubeflow pipeline: '{pipeline_name}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
f"{tip_string}"
) from ex
self.log_pipeline_info(pipeline_name, "pipeline uploaded", duration=time.time() - t0)
#############
# Create Experiment
#############
try:
t0 = time.time()
# create a new experiment (if already exists, this a no-op)
experiment = client.create_experiment(name=experiment_name, namespace=user_namespace)
except Exception as ex:
raise RuntimeError(
f"Failed to create Kubeflow experiment: '{experiment_name}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
) from ex
self.log_pipeline_info(pipeline_name, "created experiment", duration=time.time() - t0)
#############
# Create Pipeline Run
#############
try:
t0 = time.time()
# generate name for the pipeline run
job_name = f"{pipeline_name}-{timestamp}"
# create pipeline run (or specified pipeline version)
run = client.run_pipeline(
experiment_id=experiment.id, job_name=job_name, pipeline_id=pipeline_id, version_id=version_id
)
except Exception as ex:
raise RuntimeError(
f"Failed to create Kubeflow pipeline run: '{job_name}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
) from ex
if run is None:
# client.run_pipeline seemed to have encountered an issue
# but didn't raise an exception
raise RuntimeError(
f"Failed to create Kubeflow pipeline run: '{job_name}' - "
f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'"
)
self.log_pipeline_info(
pipeline_name, f"pipeline submitted: {api_endpoint}/#/runs/details/{run.id}", duration=time.time() - t0
)
return KfpPipelineProcessorResponse(
run_id=run.id,
run_url=f"{api_endpoint}/#/runs/details/{run.id}",
object_storage_url=f"{cos_endpoint}",
object_storage_path=f"/{cos_bucket}/{cos_directory}",
)
def export(self, pipeline, pipeline_export_format, pipeline_export_path,
overwrite):
# Verify that the KfpPipelineProcessor supports the given export format
self._verify_export_format(pipeline_export_format)
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = pipeline.name
# Create an instance id that will be used to store
# the pipelines' dependencies, if applicable
pipeline_instance_id = f"{pipeline_name}-{timestamp}"
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(
self.root_dir, pipeline_export_path)
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID,
name=pipeline.runtime_config)
engine = runtime_configuration.metadata.get("engine")
if engine == "Tekton" and not TektonClient:
raise ValueError(
"kfp-tekton not installed. Please install using elyra[kfp-tekton] to use Tekton engine."
)
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path +
" already exists.")
self.log_pipeline_info(
pipeline_name,
f"Exporting pipeline as a .{pipeline_export_format} file")
# Export pipeline as static configuration file (YAML formatted)
try:
# Exported pipeline is not associated with an experiment
# or a version. The association is established when the
# pipeline is imported into KFP by the user.
pipeline_function = lambda: self._cc_pipeline(
pipeline,
pipeline_name,
pipeline_instance_id=pipeline_instance_id) # nopep8
if engine == "Tekton":
self.log.info("Compiling pipeline for Tekton engine")
kfp_tekton_compiler.TektonCompiler().compile(
pipeline_function, absolute_pipeline_export_path)
else:
self.log.info("Compiling pipeline for Argo engine")
kfp_argo_compiler.Compiler().compile(
pipeline_function, absolute_pipeline_export_path)
except RuntimeError:
raise
except Exception as ex:
if ex.__cause__:
raise RuntimeError(str(ex)) from ex
raise RuntimeError(
f"Error pre-processing pipeline '{pipeline_name}' for export to '{absolute_pipeline_export_path}'",
str(ex),
) from ex
self.log_pipeline_info(
pipeline_name,
f"pipeline exported to '{pipeline_export_path}'",
duration=(time.time() - t0_all))
return pipeline_export_path # Return the input value, not its absolute form
