def export(self, pipeline, pipeline_export_format, pipeline_export_path,
overwrite):
if pipeline_export_format not in ["py"]:
raise ValueError(
"Pipeline export format {} not recognized.".format(
pipeline_export_format))
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = f'{pipeline.name}-{timestamp}'
absolute_pipeline_export_path = get_absolute_path(
self.root_dir, pipeline_export_path)
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path +
" already exists.")
self.log_pipeline_info(
pipeline_name,
f"exporting pipeline as a .{pipeline_export_format} file")
self.create_pipeline_file(pipeline=pipeline,
pipeline_export_format="py",
pipeline_export_path=pipeline_export_path,
pipeline_name=pipeline_name)
return pipeline_export_path
def get_valid_filepath(self, op_filename: str) -> str:
filepath = get_absolute_path(self._root_dir, op_filename)
if not os.path.exists(filepath):
raise FileNotFoundError(f'Could not find {filepath}')
if not os.path.isfile(filepath):
raise ValueError(f'Not a file: {filepath}')
return filepath
def export(self, pipeline, pipeline_export_format, pipeline_export_path,
overwrite):
# Verify that the AirflowPipelineProcessor supports the given export format
self._verify_export_format(pipeline_export_format)
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = f"{pipeline.name}-{timestamp}"
absolute_pipeline_export_path = get_absolute_path(
self.root_dir, pipeline_export_path)
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path +
" already exists.")
self.log_pipeline_info(
pipeline_name,
f"exporting pipeline as a .{pipeline_export_format} file")
new_pipeline_file_path = self.create_pipeline_file(
pipeline=pipeline,
pipeline_export_format="py",
pipeline_export_path=absolute_pipeline_export_path,
pipeline_name=pipeline_name,
)
return new_pipeline_file_path
def export(
self, pipeline: Pipeline, pipeline_export_format: str, pipeline_export_path: str, overwrite: bool
) -> str:
"""
Export pipeline as Airflow DAG
"""
# Verify that the AirflowPipelineProcessor supports the given export format
self._verify_export_format(pipeline_export_format)
timestamp = datetime.now().strftime("%m%d%H%M%S")
# Create an instance id that will be used to store
# the pipelines' dependencies, if applicable
pipeline_instance_id = f"{pipeline.name}-{timestamp}"
absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path)
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError(f"File '{absolute_pipeline_export_path}' already exists.")
self.log_pipeline_info(pipeline.name, f"exporting pipeline as a .{pipeline_export_format} file")
new_pipeline_file_path = self.create_pipeline_file(
pipeline=pipeline,
pipeline_export_format="py",
pipeline_export_path=absolute_pipeline_export_path,
pipeline_name=pipeline.name,
pipeline_instance_id=pipeline_instance_id,
)
return new_pipeline_file_path
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite):
# Verify that the KfpPipelineProcessor supports the given export format
self._verify_export_format(pipeline_export_format)
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = pipeline.name
# Unique location on COS where the pipeline run artifacts
# will be stored
cos_directory = f"{pipeline_name}-{timestamp}"
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path)
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID, name=pipeline.runtime_config
)
engine = runtime_configuration.metadata.get("engine")
if engine == "Tekton" and not TektonClient:
raise ValueError("kfp-tekton not installed. Please install using elyra[kfp-tekton] to use Tekton engine.")
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path + " already exists.")
self.log_pipeline_info(pipeline_name, f"Exporting pipeline as a .{pipeline_export_format} file")
# Export pipeline as static configuration file (YAML formatted)
try:
# Exported pipeline is not associated with an experiment
# or a version. The association is established when the
# pipeline is imported into KFP by the user.
pipeline_function = lambda: self._cc_pipeline(
pipeline, pipeline_name, cos_directory=cos_directory
) # nopep8
if engine == "Tekton":
self.log.info("Compiling pipeline for Tekton engine")
kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, absolute_pipeline_export_path)
else:
self.log.info("Compiling pipeline for Argo engine")
kfp_argo_compiler.Compiler().compile(pipeline_function, absolute_pipeline_export_path)
except RuntimeError:
raise
except Exception as ex:
if ex.__cause__:
raise RuntimeError(str(ex)) from ex
raise RuntimeError(
f"Error pre-processing pipeline {pipeline_name} for export at {absolute_pipeline_export_path}",
str(ex),
) from ex
self.log_pipeline_info(
pipeline_name, f"pipeline exported: {pipeline_export_path}", duration=(time.time() - t0_all)
)
return pipeline_export_path # Return the input value, not its absolute form
def process(self, operation: Operation):
filepath = get_absolute_path(self._root_dir, operation.filename)
if not os.path.exists(filepath):
raise FileNotFoundError(f'Could not find {filepath}')
if not os.path.isfile(filepath):
raise ValueError(f'Not a file: {filepath}')
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing: {filepath}')
argv = self._create_execute_command(filepath, file_dir)
envs = operation.env_vars_as_dict
t0 = time.time()
try:
subprocess.run(argv, cwd=file_dir, env=envs, check=True)
except Exception as ex:
self.log.error(f'Internal error executing {filepath}')
raise RuntimeError(f'Internal error executing {filepath}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite):
if pipeline_export_format not in ["yaml", "py"]:
raise ValueError("Pipeline export format {} not recognized.".format(pipeline_export_format))
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = pipeline.name
pipeline_version_name = f'{pipeline_name}-{timestamp}'
# work around https://github.com/kubeflow/pipelines/issues/5172
experiment_name = pipeline_name.lower()
# Unique identifier for the pipeline run
job_name = f'{pipeline_name}-{timestamp}'
# Unique location on COS where the pipeline run artifacts
# will be stored
cos_directory = f'{pipeline_name}-{timestamp}'
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path)
runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata['api_endpoint']
namespace = runtime_configuration.metadata.get('user_namespace')
engine = runtime_configuration.metadata.get('engine')
cos_secret = runtime_configuration.metadata.get('cos_secret')
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path + " already exists.")
self.log_pipeline_info(pipeline_name, f"exporting pipeline as a .{pipeline_export_format} file")
if pipeline_export_format != "py":
# Export pipeline as static configuration file (YAML formatted)
try:
# Exported pipeline is not associated with an experiment
# or a version. The association is established when the
# pipeline is imported into KFP by the user.
pipeline_function = lambda: self._cc_pipeline(pipeline,
pipeline_name,
cos_directory=cos_directory) # nopep8
if 'Tekton' == engine:
self.log.info("Compiling pipeline for Tekton engine")
kfp_tekton_compiler.TektonCompiler().compile(pipeline_function, absolute_pipeline_export_path)
else:
self.log.info("Compiling pipeline for Argo engine")
kfp_argo_compiler.Compiler().compile(pipeline_function, absolute_pipeline_export_path)
except Exception as ex:
if ex.__cause__:
raise RuntimeError(str(ex)) from ex
raise RuntimeError('Error pre-processing pipeline {} for export at {}'.
format(pipeline_name, absolute_pipeline_export_path), str(ex)) from ex
else:
# Export pipeline as Python DSL
# Load template from installed elyra package
loader = PackageLoader('elyra', 'templates/kfp')
template_env = Environment(loader=loader, trim_blocks=True)
template_env.filters['to_basename'] = lambda path: os.path.basename(path)
template = template_env.get_template('kfp_template.jinja2')
defined_pipeline = self._cc_pipeline(pipeline,
pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
cos_directory=cos_directory,
export=True)
description = f'Created with Elyra {__version__} pipeline editor using {pipeline.source}.'
for key, operation in defined_pipeline.items():
self.log.debug("component :\n "
"container op name : %s \n "
"inputs : %s \n "
"outputs : %s \n ",
operation.name,
operation.inputs,
operation.outputs)
# The exported pipeline is by default associated with
# an experiment.
# The user can manually customize the generated code
# and change the associations as desired.
python_output = template.render(operations_list=defined_pipeline,
pipeline_name=pipeline_name,
pipeline_version=pipeline_version_name,
experiment_name=experiment_name,
run_name=job_name,
engine=engine,
cos_secret=cos_secret,
namespace=namespace,
api_endpoint=api_endpoint,
pipeline_description=description,
writable_container_dir=self.WCD)
# Write to Python file and fix formatting
with open(absolute_pipeline_export_path, "w") as fh:
autopep_output = autopep8.fix_code(python_output)
output_to_file = format_str(autopep_output, mode=FileMode())
fh.write(output_to_file)
self.log_pipeline_info(pipeline_name, "pipeline rendered", duration=(time.time() - t0_all))
self.log_pipeline_info(pipeline_name,
f"pipeline exported: {pipeline_export_path}",
duration=(time.time() - t0_all))
return pipeline_export_path # Return the input value, not its absolute form
def export(self, pipeline, pipeline_export_format, pipeline_export_path, overwrite):
if pipeline_export_format not in ["yaml", "py"]:
raise ValueError("Pipeline export format {} not recognized.".format(pipeline_export_format))
t0_all = time.time()
pipeline_name = pipeline.name
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(self.root_dir, pipeline_export_path)
runtime_configuration = self._get_runtime_configuration(pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata['api_endpoint']
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path + " already exists.")
self.log_pipeline_info(pipeline_name, f"exporting pipeline as a .{pipeline_export_format} file")
if pipeline_export_format != "py":
try:
pipeline_function = lambda: self._cc_pipeline(pipeline, pipeline_name) # nopep8
kfp.compiler.Compiler().compile(pipeline_function, absolute_pipeline_export_path)
except Exception as ex:
raise RuntimeError('Error compiling pipeline {} for export at {}'.
format(pipeline_name, absolute_pipeline_export_path), str(ex)) from ex
else:
# Load template from installed elyra package
t0 = time.time()
loader = PackageLoader('elyra', 'templates')
template_env = Environment(loader=loader)
template_env.filters['to_basename'] = lambda path: os.path.basename(path)
template = template_env.get_template('kfp_template.jinja2')
defined_pipeline = self._cc_pipeline(pipeline, pipeline_name)
description = f'Created with Elyra {__version__} pipeline editor using {pipeline.name}.pipeline.'
for key, operation in defined_pipeline.items():
self.log.debug("component :\n "
"container op name : %s \n "
"inputs : %s \n "
"outputs : %s \n ",
operation.name,
operation.inputs,
operation.outputs)
python_output = template.render(operations_list=defined_pipeline,
pipeline_name=pipeline_name,
api_endpoint=api_endpoint,
pipeline_description=description,
writable_container_dir=self.WCD)
# Write to python file and fix formatting
with open(absolute_pipeline_export_path, "w") as fh:
fh.write(autopep8.fix_code(python_output))
self.log_pipeline_info(pipeline_name, "pipeline rendered", duration=(time.time() - t0))
self.log_pipeline_info(pipeline_name,
f"pipeline exported: {pipeline_export_path}",
duration=(time.time() - t0_all))
return pipeline_export_path # Return the input value, not its absolute form
def export(self, pipeline, pipeline_export_format, pipeline_export_path,
overwrite):
if pipeline_export_format not in ["yaml", "py"]:
raise ValueError(
"Pipeline export format {} not recognized.".format(
pipeline_export_format))
pipeline_name = pipeline.name
# Since pipeline_export_path may be relative to the notebook directory, ensure
# we're using its absolute form.
absolute_pipeline_export_path = get_absolute_path(
self.root_dir, pipeline_export_path)
runtime_configuration = self._get_runtime_configuration(
pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata['api_endpoint']
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path +
" already exists.")
self.log.info('Creating pipeline definition as a .' +
pipeline_export_format + ' file')
if pipeline_export_format != "py":
try:
pipeline_function = lambda: self._cc_pipeline(
pipeline, pipeline_name) # nopep8
t0 = time.time()
kfp.compiler.Compiler().compile(pipeline_function,
absolute_pipeline_export_path)
t1 = time.time()
self.log.debug(
"Compilation of pipeline '{name}' took {duration:.3f} secs."
.format(name=pipeline_name, duration=(t1 - t0)))
except Exception as ex:
raise RuntimeError(
'Error compiling pipeline {} for export at {}'.format(
pipeline_name, absolute_pipeline_export_path),
str(ex)) from ex
else:
# Load template from installed elyra package
loader = PackageLoader('elyra', 'templates')
template_env = Environment(loader=loader)
template = template_env.get_template('kfp_template.jinja2')
defined_pipeline = self._cc_pipeline(pipeline, pipeline_name)
for key, operation in defined_pipeline.items():
self.log.debug(
"component :\n "
"container op name : %s \n "
"inputs : %s \n "
"outputs : %s \n ", operation.name, operation.inputs,
operation.outputs)
python_output = template.render(
operations_list=defined_pipeline,
pipeline_name=pipeline_name,
api_endpoint=api_endpoint,
pipeline_description="Elyra Pipeline")
# Write to python file and fix formatting
with open(absolute_pipeline_export_path, "w") as fh:
fh.write(autopep8.fix_code(python_output))
return pipeline_export_path # Return the input value, not its absolute form
