def test_recur_nested_separate_workflow(self):
"""
Test compiling a nested recursive workflow with embedded loop resource in annotations.
"""
from .testdata.recur_nested import flipcoin
pipeline_conf = TektonPipelineConf()
pipeline_conf.set_resource_in_separate_yaml(True)
self._test_pipeline_workflow(flipcoin,
'recur_nested_separate.yaml',
tekton_pipeline_conf=pipeline_conf)
def _test_pipeline_workflow(self,
pipeline_function,
pipeline_yaml,
normalize_compiler_output_function=None,
tekton_pipeline_conf=TektonPipelineConf(),
skip_noninlined=False):
self._test_pipeline_workflow_inlined_spec(
pipeline_function=pipeline_function,
pipeline_yaml=pipeline_yaml,
normalize_compiler_output_function=
normalize_compiler_output_function,
tekton_pipeline_conf=tekton_pipeline_conf)
if not skip_noninlined:
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_yaml_file = os.path.join(
test_data_dir,
pipeline_yaml.replace(".yaml", "") + "_noninlined.yaml")
temp_dir = tempfile.mkdtemp()
compiled_yaml_file = os.path.join(temp_dir, 'workflow.yaml')
tekton_pipeline_conf.set_tekton_inline_spec(False)
try:
compiler.TektonCompiler().compile(
pipeline_function,
compiled_yaml_file,
tekton_pipeline_conf=tekton_pipeline_conf)
with open(compiled_yaml_file, 'r') as f:
f = normalize_compiler_output_function(
f.read()) if normalize_compiler_output_function else f
compiled = yaml.safe_load(f)
self._verify_compiled_workflow(golden_yaml_file, compiled)
finally:
shutil.rmtree(temp_dir)
def test_tekton_pipeline_conf(self):
"""
Test applying Tekton pipeline config to a workflow
"""
from .testdata.tekton_pipeline_conf import echo_pipeline
pipeline_conf = TektonPipelineConf()
pipeline_conf.add_pipeline_label('test', 'label')
pipeline_conf.add_pipeline_label('test2', 'label2')
pipeline_conf.add_pipeline_annotation('test', 'annotation')
self._test_pipeline_workflow(echo_pipeline,
'tekton_pipeline_conf.yaml',
tekton_pipeline_conf=pipeline_conf)
def compile_pipeline(
*, # force kwargs for time_it decorator to get pipeline_name
pipeline_name: str,
pipeline_script: Path) -> str:
file_name = pipeline_name + '.yaml'
tmpdir = tempfile.gettempdir() # TODO: keep compiled pipelines?
pipeline_package_path = os.path.join(tmpdir, file_name)
pipeline_conf = TektonPipelineConf()
try:
compile_pyfile(pyfile=pipeline_script,
function_name=None,
output_path=pipeline_package_path,
type_check=True,
tekton_pipeline_conf=pipeline_conf)
except ValueError as e:
print(
f"{e.__class__.__name__} trying to compile {pipeline_script}: {str(e)}"
)
# TODO: delete those files after running test or keep for inspection?
return pipeline_package_path
def main():
print(f"KFP-Tekton Compiler {__version__}")
args = parse_arguments()
if ((args.py is None and args.package is None)
or (args.py is not None and args.package is not None)):
raise ValueError('Either --py or --package is needed but not both.')
if args.py:
tekton_pipeline_conf = TektonPipelineConf()
tekton_pipeline_conf.set_tekton_inline_spec(True)
if args.disable_task_inline:
tekton_pipeline_conf.set_tekton_inline_spec(False)
compile_pyfile(args.py, args.function, args.output,
not args.disable_type_check, tekton_pipeline_conf)
else:
if args.namespace is None:
raise ValueError('--namespace is required for compiling packages.')
# TODO: deprecated, but without the monkey-patch this will produce Argo YAML
kfp_compiler_main.compile_package(args.package, args.namespace,
args.function, args.output,
not args.disable_type_check)
image: library/bash:4.4.23
command: ['sh', '-c']
args:
- |
set -e
echo op1 "$0" "$1"
- {inputValue: item}
- {inputValue: param}
'''
@dsl.pipeline(name='pipeline')
def pipeline(param: int = 10):
loop_args = [1, 2]
with dsl.ParallelFor(loop_args, parallelism=1) as item:
op1_template = components.load_component_from_text(op1_yaml)
op1 = op1_template(item, param)
condi_1 = tekton.CEL_ConditionOp(f"{item} == 2").output
with dsl.Condition(condi_1 == 'true'):
tekton.Break()
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf
tekton_pipeline_conf = TektonPipelineConf()
tekton_pipeline_conf.set_tekton_inline_spec(True)
Compiler().compile(pipeline,
__file__.replace('.py', '.yaml'),
tekton_pipeline_conf=tekton_pipeline_conf)
set -e
echo $0 > $1
- {inputValue: input_text}
- {outputPath: output_value}
""" % (name))
return print_op(msg)
@dsl.graph_component
def function_the_name_of_which_is_exactly_51_chars_long(i: int):
decr_i = CEL_ConditionOp(f"{i} - 1").output
PrintOp("print-iter", f"Iter: {decr_i}")
with dsl.Condition(decr_i != 0):
function_the_name_of_which_is_exactly_51_chars_long(decr_i)
@dsl.pipeline("pipeline-the-name-of-which-is-exactly-51-chars-long")
def pipeline_the_name_of_which_is_exactly_51_chars_long(iter_num: int = 42):
function_the_name_of_which_is_exactly_51_chars_long(iter_num)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf
tekton_pipeline_conf = TektonPipelineConf()
tekton_pipeline_conf.set_tekton_inline_spec(False)
tekton_pipeline_conf.set_resource_in_separate_yaml(True)
Compiler().compile(pipeline_the_name_of_which_is_exactly_51_chars_long,
__file__.replace('.py', '.yaml'),
tekton_pipeline_conf=tekton_pipeline_conf)