def test_workflow_without_decorator(pipeline_mod, params_dict): """Test compiling a workflow and appending pipeline params.""" try: pipeline_params = [] for param in params_dict.get('paramsList', []): pipeline_params.append(getattr(pipeline_mod, param)) compiled_workflow = compiler.TektonCompiler()._create_workflow( getattr(pipeline_mod, params_dict['function']), params_dict.get('name', None), params_dict.get('description', None), pipeline_params if pipeline_params else None, params_dict.get('conf', None)) return True except: return False
def _test_pipeline_workflow(self, pipeline_function, pipeline_yaml, normalize_compiler_output_function=None): test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml) temp_dir = tempfile.mkdtemp() compiled_yaml_file = os.path.join(temp_dir, 'workflow.yaml') try: compiler.TektonCompiler().compile(pipeline_function, compiled_yaml_file) with open(compiled_yaml_file, 'r') as f: f = normalize_compiler_output_function( f.read()) if normalize_compiler_output_function else f compiled = yaml.safe_load(f) self._verify_compiled_workflow(golden_yaml_file, compiled) finally: shutil.rmtree(temp_dir)
def _test_workflow_without_decorator(self, pipeline_yaml, params_dict): """ Test compiling a workflow and appending pipeline params. """ test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml) temp_dir = tempfile.mkdtemp() try: kfp_tekton_compiler = compiler.TektonCompiler() kfp_tekton_compiler.generate_pipelinerun = True compiled_workflow = kfp_tekton_compiler._create_workflow( params_dict['function'], params_dict.get('name', None), params_dict.get('description', None), params_dict.get('paramsList', None), params_dict.get('conf', None)) self._verify_compiled_workflow(golden_yaml_file, compiled_workflow) finally: shutil.rmtree(temp_dir)
def _test_nested_workflow(self, pipeline_yaml, pipeline_list, normalize_compiler_output_function=None): """ Test compiling a simple workflow, and a bigger one composed from the simple one. """ test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml) temp_dir = tempfile.mkdtemp() compiled_yaml_file = os.path.join(temp_dir, 'nested' + str(len(pipeline_list) - 1) + '.yaml') try: for index, pipeline in enumerate(pipeline_list): package_path = os.path.join(temp_dir, 'nested' + str(index) + '.yaml') compiler.TektonCompiler().compile(pipeline, package_path) with open(compiled_yaml_file, 'r') as f: f = normalize_compiler_output_function( f.read()) if normalize_compiler_output_function else f compiled = yaml.safe_load(f) self._verify_compiled_workflow(golden_yaml_file, compiled) finally: shutil.rmtree(temp_dir)
def test_sequential_workflow(self): """ Test compiling a sequential workflow. """ test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_yaml_file = os.path.join(test_data_dir, 'sequential.yaml') temp_dir = tempfile.mkdtemp() compiled_yaml_file = os.path.join(temp_dir, 'workflow.yaml') try: from .testdata.sequential import sequential_pipeline compiler.TektonCompiler().compile(sequential_pipeline, compiled_yaml_file) with open(compiled_yaml_file, 'r') as f: compiled = list(yaml.safe_load_all(f)) if GENERATE_GOLDEN_YAML: with open(golden_yaml_file, 'w') as f: yaml.dump_all(compiled, f, default_flow_style=False) else: with open(golden_yaml_file, 'r') as f: golden = list(yaml.safe_load_all(f)) self.maxDiff = None self.assertEqual(golden, compiled) finally: shutil.rmtree(temp_dir)
def process(self, pipeline): """Runs a pipeline on Kubeflow Pipelines Each time a pipeline is processed, a new version is uploaded and run under the same experiment name. """ t0_all = time.time() timestamp = datetime.now().strftime("%m%d%H%M%S") runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES, name=pipeline.runtime_config) api_endpoint = runtime_configuration.metadata['api_endpoint'] cos_endpoint = runtime_configuration.metadata['cos_endpoint'] cos_bucket = runtime_configuration.metadata['cos_bucket'] user_namespace = runtime_configuration.metadata.get('user_namespace') # TODO: try to encapsulate the info below api_username = runtime_configuration.metadata.get('api_username') api_password = runtime_configuration.metadata.get('api_password') engine = runtime_configuration.metadata.get('engine') pipeline_name = pipeline.name try: # Connect to the Kubeflow server, determine whether it is secured, # and if it is try to authenticate with the user-provided credentials # (if any were defined in the runtime configuration) endpoint = api_endpoint.replace('/pipeline', '') auth_info = \ KfpPipelineProcessor._get_user_auth_session_cookie(endpoint, api_username, api_password) self.log.debug(f"Kubeflow authentication info: {auth_info}") if auth_info['endpoint_secured'] and \ auth_info['authservice_session_cookie'] is None: # Kubeflow is secured but our attempt to authenticate did # not yield the expected results. Log the collected authentication # information and abort processing. self.log.warning(f"Kubeflow authentication info: {auth_info}") raise RuntimeError(f"Error connecting to Kubeflow at '{endpoint}'" f": Authentication request failed. Check the " f"Kubeflow Pipelines credentials in runtime " f"configuration '{pipeline.runtime_config}'.") # Create a KFP client if 'Tekton' == engine: client = TektonClient(host=api_endpoint, cookies=auth_info['authservice_session_cookie']) else: client = ArgoClient(host=api_endpoint, cookies=auth_info['authservice_session_cookie']) # Determine whether a pipeline with the provided # name already exists pipeline_id = client.get_pipeline_id(pipeline_name) if pipeline_id is None: # The KFP default version name is the pipeline # name pipeline_version_name = pipeline_name else: # Append timestamp to generate unique version name pipeline_version_name = f'{pipeline_name}-{timestamp}' # Establish a 1:1 relationship with an experiment # work around https://github.com/kubeflow/pipelines/issues/5172 experiment_name = pipeline_name.lower() # Unique identifier for the pipeline run job_name = f'{pipeline_name}-{timestamp}' # Unique location on COS where the pipeline run artifacts # will be stored cos_directory = f'{pipeline_name}-{timestamp}' except MaxRetryError as ex: raise RuntimeError('Error connecting to pipeline server {}'.format(api_endpoint)) from ex except LocationValueError as lve: if api_username: raise ValueError("Failure occurred uploading pipeline, check your credentials") from lve else: raise lve # Verify that user-entered namespace is valid try: client.list_experiments(namespace=user_namespace, page_size=0) except ApiException as ae: error_msg = f"{ae.reason} ({ae.status})" if ae.body: error_body = json.loads(ae.body) error_msg += f": {error_body['error']}" if error_msg[-1] not in ['.', '?', '!']: error_msg += '.' namespace = "namespace" if not user_namespace else f"namespace {user_namespace}" self.log.error(f"Error validating {namespace}: {error_msg}") raise RuntimeError(f"Error validating {namespace}: {error_msg} " + "Please validate your runtime configuration details and retry.") from ae self.log_pipeline_info(pipeline_name, "submitting pipeline") with tempfile.TemporaryDirectory() as temp_dir: pipeline_path = os.path.join(temp_dir, f'{pipeline_name}.tar.gz') self.log.debug("Creating temp directory %s", temp_dir) # Compile the new pipeline try: pipeline_function = lambda: self._cc_pipeline(pipeline, # nopep8 E731 pipeline_name=pipeline_name, pipeline_version=pipeline_version_name, experiment_name=experiment_name, cos_directory=cos_directory) if 'Tekton' == engine: kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, pipeline_path) else: kfp_argo_compiler.Compiler().compile(pipeline_function, pipeline_path) except Exception as ex: if ex.__cause__: raise RuntimeError(str(ex)) from ex raise RuntimeError('Error pre-processing pipeline {} for engine {} at {}'. format(pipeline_name, engine, pipeline_path), str(ex)) from ex self.log.debug("Kubeflow Pipeline was created in %s", pipeline_path) # Upload the compiled pipeline, create an experiment and run try: description = f"Created with Elyra {__version__} pipeline editor using '{pipeline.source}'." t0 = time.time() if pipeline_id is None: # Upload new pipeline. The call returns # a unique pipeline id. kfp_pipeline = \ client.upload_pipeline(pipeline_path, pipeline_name, description) pipeline_id = kfp_pipeline.id version_id = None else: # Upload a pipeline version. The call returns # a unique version id. kfp_pipeline = \ client.upload_pipeline_version(pipeline_path, pipeline_version_name, pipeline_id=pipeline_id) version_id = kfp_pipeline.id self.log_pipeline_info(pipeline_name, 'pipeline uploaded', duration=(time.time() - t0)) except MaxRetryError as ex: raise RuntimeError('Error connecting to pipeline server {}'.format(api_endpoint)) from ex except LocationValueError as lve: if api_username: raise ValueError("Failure occurred uploading pipeline, check your credentials") from lve else: raise lve # Create a new experiment. If it already exists this is # a no-op. experiment = client.create_experiment(name=experiment_name, namespace=user_namespace) self.log_pipeline_info(pipeline_name, f'Created experiment {experiment_name}', duration=(time.time() - t0_all)) # Run the pipeline (or specified pipeline version) run = client.run_pipeline(experiment_id=experiment.id, job_name=job_name, pipeline_id=pipeline_id, version_id=version_id) self.log_pipeline_info(pipeline_name, f"pipeline submitted: {api_endpoint}/#/runs/details/{run.id}", duration=(time.time() - t0_all)) return KfpPipelineProcessorResponse( run_url=f'{api_endpoint}/#/runs/details/{run.id}', object_storage_url=f'{cos_endpoint}', object_storage_path=f'/{cos_bucket}/{cos_directory}', ) return None
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite): if pipeline_export_format not in ["yaml", "py"]: raise ValueError("Pipeline export format {} not recognized.".format(pipeline_export_format)) t0_all = time.time() timestamp = datetime.now().strftime("%m%d%H%M%S") pipeline_name = pipeline.name pipeline_version_name = f'{pipeline_name}-{timestamp}' # work around https://github.com/kubeflow/pipelines/issues/5172 experiment_name = pipeline_name.lower() # Unique identifier for the pipeline run job_name = f'{pipeline_name}-{timestamp}' # Unique location on COS where the pipeline run artifacts # will be stored cos_directory = f'{pipeline_name}-{timestamp}' # Since pipeline_export_path may be relative to the notebook directory, ensure # we're using its absolute form. absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path) runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES, name=pipeline.runtime_config) api_endpoint = runtime_configuration.metadata['api_endpoint'] namespace = runtime_configuration.metadata.get('user_namespace') engine = runtime_configuration.metadata.get('engine') cos_secret = runtime_configuration.metadata.get('cos_secret') if os.path.exists(absolute_pipeline_export_path) and not overwrite: raise ValueError("File " + absolute_pipeline_export_path + " already exists.") self.log_pipeline_info(pipeline_name, f"exporting pipeline as a .{pipeline_export_format} file") if pipeline_export_format != "py": # Export pipeline as static configuration file (YAML formatted) try: # Exported pipeline is not associated with an experiment # or a version. The association is established when the # pipeline is imported into KFP by the user. pipeline_function = lambda: self._cc_pipeline(pipeline, pipeline_name, cos_directory=cos_directory) # nopep8 if 'Tekton' == engine: self.log.info("Compiling pipeline for Tekton engine") kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, absolute_pipeline_export_path) else: self.log.info("Compiling pipeline for Argo engine") kfp_argo_compiler.Compiler().compile(pipeline_function, absolute_pipeline_export_path) except Exception as ex: if ex.__cause__: raise RuntimeError(str(ex)) from ex raise RuntimeError('Error pre-processing pipeline {} for export at {}'. format(pipeline_name, absolute_pipeline_export_path), str(ex)) from ex else: # Export pipeline as Python DSL # Load template from installed elyra package loader = PackageLoader('elyra', 'templates/kfp') template_env = Environment(loader=loader, trim_blocks=True) template_env.filters['to_basename'] = lambda path: os.path.basename(path) template = template_env.get_template('kfp_template.jinja2') defined_pipeline = self._cc_pipeline(pipeline, pipeline_name, pipeline_version=pipeline_version_name, experiment_name=experiment_name, cos_directory=cos_directory, export=True) description = f'Created with Elyra {__version__} pipeline editor using {pipeline.source}.' for key, operation in defined_pipeline.items(): self.log.debug("component :\n " "container op name : %s \n " "inputs : %s \n " "outputs : %s \n ", operation.name, operation.inputs, operation.outputs) # The exported pipeline is by default associated with # an experiment. # The user can manually customize the generated code # and change the associations as desired. python_output = template.render(operations_list=defined_pipeline, pipeline_name=pipeline_name, pipeline_version=pipeline_version_name, experiment_name=experiment_name, run_name=job_name, engine=engine, cos_secret=cos_secret, namespace=namespace, api_endpoint=api_endpoint, pipeline_description=description, writable_container_dir=self.WCD) # Write to Python file and fix formatting with open(absolute_pipeline_export_path, "w") as fh: autopep_output = autopep8.fix_code(python_output) output_to_file = format_str(autopep_output, mode=FileMode()) fh.write(output_to_file) self.log_pipeline_info(pipeline_name, "pipeline rendered", duration=(time.time() - t0_all)) self.log_pipeline_info(pipeline_name, f"pipeline exported: {pipeline_export_path}", duration=(time.time() - t0_all)) return pipeline_export_path # Return the input value, not its absolute form
def process(self, pipeline): """ Runs a pipeline on Kubeflow Pipelines Each time a pipeline is processed, a new version is uploaded and run under the same experiment name. """ timestamp = datetime.now().strftime("%m%d%H%M%S") ################ # Runtime Configs ################ runtime_configuration = self._get_metadata_configuration( schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID, name=pipeline.runtime_config ) # unpack Kubeflow Pipelines configs api_endpoint = runtime_configuration.metadata["api_endpoint"].rstrip("/") api_username = runtime_configuration.metadata.get("api_username") api_password = runtime_configuration.metadata.get("api_password") user_namespace = runtime_configuration.metadata.get("user_namespace") engine = runtime_configuration.metadata.get("engine") if engine == "Tekton" and not TektonClient: raise ValueError( "Python package `kfp-tekton` is not installed. " "Please install using `elyra[kfp-tekton]` to use Tekton engine." ) # unpack Cloud Object Storage configs cos_endpoint = runtime_configuration.metadata["cos_endpoint"] cos_bucket = runtime_configuration.metadata["cos_bucket"] # Determine which provider to use to authenticate with Kubeflow auth_type = runtime_configuration.metadata.get("auth_type") try: auth_info = KFPAuthenticator().authenticate( api_endpoint, auth_type_str=auth_type, runtime_config_name=pipeline.runtime_config, auth_parm_1=api_username, auth_parm_2=api_password, ) self.log.debug(f"Authenticator returned {auth_info}") except AuthenticationError as ae: if ae.get_request_history() is not None: self.log.info("An authentication error was raised. Diagnostic information follows.") self.log.info(ae.request_history_to_string()) raise RuntimeError(f"Kubeflow authentication failed: {ae}") ############# # Create Kubeflow Client ############# try: if engine == "Tekton": client = TektonClient( host=api_endpoint, cookies=auth_info.get("cookies", None), credentials=auth_info.get("credentials", None), existing_token=auth_info.get("existing_token", None), namespace=user_namespace, ) else: client = ArgoClient( host=api_endpoint, cookies=auth_info.get("cookies", None), credentials=auth_info.get("credentials", None), existing_token=auth_info.get("existing_token", None), namespace=user_namespace, ) except Exception as ex: # a common cause of these errors is forgetting to include `/pipeline` or including it with an 's' api_endpoint_obj = urlsplit(api_endpoint) if api_endpoint_obj.path != "/pipeline": api_endpoint_tip = api_endpoint_obj._replace(path="/pipeline").geturl() tip_string = ( f" - [TIP: did you mean to set '{api_endpoint_tip}' as the endpoint, " f"take care not to include 's' at end]" ) else: tip_string = "" raise RuntimeError( f"Failed to initialize `kfp.Client()` against: '{api_endpoint}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" f"{tip_string}" ) from ex ############# # Verify Namespace ############# try: client.list_experiments(namespace=user_namespace, page_size=1) except Exception as ex: if user_namespace: tip_string = f"[TIP: ensure namespace '{user_namespace}' is correct]" else: tip_string = "[TIP: you probably need to set a namespace]" raise RuntimeError( f"Failed to `kfp.Client().list_experiments()` against: '{api_endpoint}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}' - " f"{tip_string}" ) from ex ############# # Pipeline Metadata ############# # generate a pipeline name pipeline_name = pipeline.name # generate a pipeline description pipeline_description = pipeline.description if pipeline_description is None: pipeline_description = f"Created with Elyra {__version__} pipeline editor using `{pipeline.source}`." ############# # Submit & Run the Pipeline ############# self.log_pipeline_info(pipeline_name, "submitting pipeline") with tempfile.TemporaryDirectory() as temp_dir: self.log.debug(f"Created temporary directory at: {temp_dir}") pipeline_path = os.path.join(temp_dir, f"{pipeline_name}.tar.gz") ############# # Get Pipeline ID ############# try: # get the kubeflow pipeline id (returns None if not found, otherwise the ID of the pipeline) pipeline_id = client.get_pipeline_id(pipeline_name) # calculate what "pipeline version" name to use if pipeline_id is None: # the first "pipeline version" name must be the pipeline name pipeline_version_name = pipeline_name else: # generate a unique name for a new "pipeline version" by appending the current timestamp pipeline_version_name = f"{pipeline_name}-{timestamp}" except Exception as ex: raise RuntimeError( f"Failed to get ID of Kubeflow pipeline: '{pipeline_name}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" ) from ex ############# # Compile the Pipeline ############# try: t0 = time.time() # generate a name for the experiment (lowercase because experiments are case intensive) experiment_name = pipeline_name.lower() # unique location on COS where the pipeline run artifacts will be stored cos_directory = f"{pipeline_name}-{timestamp}" pipeline_function = lambda: self._cc_pipeline( # nopep8 E731 pipeline, pipeline_name=pipeline_name, pipeline_version=pipeline_version_name, experiment_name=experiment_name, cos_directory=cos_directory, ) # collect pipeline configuration information pipeline_conf = self._generate_pipeline_conf(pipeline) # compile the pipeline if engine == "Tekton": kfp_tekton_compiler.TektonCompiler().compile( pipeline_function, pipeline_path, pipeline_conf=pipeline_conf ) else: kfp_argo_compiler.Compiler().compile(pipeline_function, pipeline_path, pipeline_conf=pipeline_conf) except RuntimeError: raise except Exception as ex: raise RuntimeError( f"Failed to compile pipeline '{pipeline_name}' with engine '{engine}' to: '{pipeline_path}'" ) from ex self.log_pipeline_info(pipeline_name, "pipeline compiled", duration=time.time() - t0) ############# # Upload Pipeline Version ############# try: t0 = time.time() # CASE 1: pipeline needs to be created if pipeline_id is None: # create new pipeline (and initial "pipeline version") kfp_pipeline = client.upload_pipeline( pipeline_package_path=pipeline_path, pipeline_name=pipeline_name, description=pipeline_description, ) # extract the ID of the pipeline we created pipeline_id = kfp_pipeline.id # the initial "pipeline version" has the same id as the pipeline itself version_id = pipeline_id # CASE 2: pipeline already exists else: # upload the "pipeline version" kfp_pipeline = client.upload_pipeline_version( pipeline_package_path=pipeline_path, pipeline_version_name=pipeline_version_name, pipeline_id=pipeline_id, ) # extract the id of the "pipeline version" that was created version_id = kfp_pipeline.id except Exception as ex: # a common cause of these errors is forgetting to include `/pipeline` or including it with an 's' api_endpoint_obj = urlsplit(api_endpoint) if api_endpoint_obj.path != "/pipeline": api_endpoint_tip = api_endpoint_obj._replace(path="/pipeline").geturl() tip_string = ( f" - [TIP: did you mean to set '{api_endpoint_tip}' as the endpoint, " f"take care not to include 's' at end]" ) else: tip_string = "" raise RuntimeError( f"Failed to upload Kubeflow pipeline: '{pipeline_name}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" f"{tip_string}" ) from ex self.log_pipeline_info(pipeline_name, "pipeline uploaded", duration=time.time() - t0) ############# # Create Experiment ############# try: t0 = time.time() # create a new experiment (if already exists, this a no-op) experiment = client.create_experiment(name=experiment_name, namespace=user_namespace) except Exception as ex: raise RuntimeError( f"Failed to create Kubeflow experiment: '{experiment_name}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" ) from ex self.log_pipeline_info(pipeline_name, "created experiment", duration=time.time() - t0) ############# # Create Pipeline Run ############# try: t0 = time.time() # generate name for the pipeline run job_name = f"{pipeline_name}-{timestamp}" # create pipeline run (or specified pipeline version) run = client.run_pipeline( experiment_id=experiment.id, job_name=job_name, pipeline_id=pipeline_id, version_id=version_id ) except Exception as ex: raise RuntimeError( f"Failed to create Kubeflow pipeline run: '{job_name}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" ) from ex if run is None: # client.run_pipeline seemed to have encountered an issue # but didn't raise an exception raise RuntimeError( f"Failed to create Kubeflow pipeline run: '{job_name}' - " f"Check Kubeflow Pipelines runtime configuration: '{pipeline.runtime_config}'" ) self.log_pipeline_info( pipeline_name, f"pipeline submitted: {api_endpoint}/#/runs/details/{run.id}", duration=time.time() - t0 ) return KfpPipelineProcessorResponse( run_id=run.id, run_url=f"{api_endpoint}/#/runs/details/{run.id}", object_storage_url=f"{cos_endpoint}", object_storage_path=f"/{cos_bucket}/{cos_directory}", )
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite): # Verify that the KfpPipelineProcessor supports the given export format self._verify_export_format(pipeline_export_format) t0_all = time.time() timestamp = datetime.now().strftime("%m%d%H%M%S") pipeline_name = pipeline.name # Create an instance id that will be used to store # the pipelines' dependencies, if applicable pipeline_instance_id = f"{pipeline_name}-{timestamp}" # Since pipeline_export_path may be relative to the notebook directory, ensure # we're using its absolute form. absolute_pipeline_export_path = get_absolute_path( self.root_dir, pipeline_export_path) runtime_configuration = self._get_metadata_configuration( schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID, name=pipeline.runtime_config) engine = runtime_configuration.metadata.get("engine") if engine == "Tekton" and not TektonClient: raise ValueError( "kfp-tekton not installed. Please install using elyra[kfp-tekton] to use Tekton engine." ) if os.path.exists(absolute_pipeline_export_path) and not overwrite: raise ValueError("File " + absolute_pipeline_export_path + " already exists.") self.log_pipeline_info( pipeline_name, f"Exporting pipeline as a .{pipeline_export_format} file") # Export pipeline as static configuration file (YAML formatted) try: # Exported pipeline is not associated with an experiment # or a version. The association is established when the # pipeline is imported into KFP by the user. pipeline_function = lambda: self._cc_pipeline( pipeline, pipeline_name, pipeline_instance_id=pipeline_instance_id) # nopep8 if engine == "Tekton": self.log.info("Compiling pipeline for Tekton engine") kfp_tekton_compiler.TektonCompiler().compile( pipeline_function, absolute_pipeline_export_path) else: self.log.info("Compiling pipeline for Argo engine") kfp_argo_compiler.Compiler().compile( pipeline_function, absolute_pipeline_export_path) except RuntimeError: raise except Exception as ex: if ex.__cause__: raise RuntimeError(str(ex)) from ex raise RuntimeError( f"Error pre-processing pipeline '{pipeline_name}' for export to '{absolute_pipeline_export_path}'", str(ex), ) from ex self.log_pipeline_info( pipeline_name, f"pipeline exported to '{pipeline_export_path}'", duration=(time.time() - t0_all)) return pipeline_export_path # Return the input value, not its absolute form