def main(fdir: Optional[str] = None) -> List[str]: fpaths: List[str] = list() from kfp_tekton.compiler import TektonCompiler as Compiler max_length = max(lengths) op_name_base = random_chars_of_len(max_length) for length in lengths: main_pipeline = pipeline_for_len(length, op_name_base) fpath = __file__.replace('.py', f'-{length}.yaml') fname = os.path.basename(fpath) if fdir is not None: fpath = os.path.join(fdir, fname) if length > 57: # Op name cannot be more than 57 characters with pytest.raises(ValueError): Compiler().compile(main_pipeline, fpath) else: Compiler().compile(main_pipeline, fpath) fpaths.append(fpath) return fpaths
image: library/bash:4.4.23 command: ['sh', '-c'] args: - | set -e echo op1 "$0" "$1" - {inputValue: item} - {inputValue: param} ''' @dsl.pipeline(name='pipeline') def pipeline(param: int = 10): loop_args = [1, 2] with dsl.ParallelFor(loop_args, parallelism=1) as item: op1_template = components.load_component_from_text(op1_yaml) op1 = op1_template(item, param) condi_1 = tekton.CEL_ConditionOp(f"{item} == 2").output with dsl.Condition(condi_1 == 'true'): tekton.Break() if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf tekton_pipeline_conf = TektonPipelineConf() tekton_pipeline_conf.set_tekton_inline_spec(True) Compiler().compile(pipeline, __file__.replace('.py', '.yaml'), tekton_pipeline_conf=tekton_pipeline_conf)
set -e echo $0 > $1 - {inputValue: input_text} - {outputPath: output_value} """ % (name)) return print_op(msg) @dsl.graph_component def function_the_name_of_which_is_exactly_51_chars_long(i: int): decr_i = CEL_ConditionOp(f"{i} - 1").output PrintOp("print-iter", f"Iter: {decr_i}") with dsl.Condition(decr_i != 0): function_the_name_of_which_is_exactly_51_chars_long(decr_i) @dsl.pipeline("pipeline-the-name-of-which-is-exactly-51-chars-long") def pipeline_the_name_of_which_is_exactly_51_chars_long(iter_num: int = 42): function_the_name_of_which_is_exactly_51_chars_long(iter_num) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf tekton_pipeline_conf = TektonPipelineConf() tekton_pipeline_conf.set_tekton_inline_spec(False) tekton_pipeline_conf.set_resource_in_separate_yaml(True) Compiler().compile(pipeline_the_name_of_which_is_exactly_51_chars_long, __file__.replace('.py', '.yaml'), tekton_pipeline_conf=tekton_pipeline_conf)
@dsl.pipeline( name='nested-recursion-pipeline', description='shows how to use graph_component and nested recursion.') def flipcoin(maxVal: int = 12): @dsl._component.graph_component def flip_component(flip_result, maxVal): @dsl._component.graph_component def flip_component_b(flip_result_b, maxVal_b): with dsl.Condition(flip_result_b == 'heads'): print_flip_b = print_op(flip_result_b) flipB = flip_coin_op().after(print_flip_b) flip_component_b(flipB.output, maxVal_b) with dsl.Condition(flip_result == 'heads'): flip_component_b(flip_result, maxVal) with dsl.Condition(flip_result == 'tails'): print_flip = print_op(flip_result) flipA = flip_coin_op().after(print_flip) flip_component(flipA.output, maxVal) flip_out = flip_coin_op() flip_loop = flip_component(flip_out.output, maxVal) print_op('cool, it is over. %s' % flip_out.output).after(flip_loop) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler # For Argo, uncomment the line below # from kfp.compiler import Compiler Compiler().compile(flipcoin, __file__.replace('.py', '.yaml'))
def gcs_download_op(url): return dsl.ContainerOp( name='GCS - Download' * 4, image='google/cloud-sdk:279.0.0', command=['sh', '-c'], arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], file_outputs={ 'data' * 10: '/tmp/results.txt', }) class PrintOp(dsl.ContainerOp): def __init__(self, name, msg): super(PrintOp, self).__init__(name=name, image='alpine:3.6', command=['echo', msg]) @dsl.pipeline( name="Some very long name with lots of words in it. " + "It should be over 63 chars long in order to observe the problem.") def main_fn(url1='gs://ml-pipeline-playground/shakespeare1.txt'): download1_task = gcs_download_op(url1) PrintOp('print' * 10, download1_task.output) if __name__ == '__main__': Compiler().compile(main_fn, __file__.replace('.py', '.yaml'))
return CEL_ConditionOp(expr) @dsl.pipeline("double-recursion-test") def double_recursion_test(until_a: int = 4, until_b: int = 3): @dsl.graph_component def recur_a(i: int, until_a: int): @dsl.graph_component def recur_b(j: int, until_b: int): print_op = PrintOp(f"Iter A: {i}, B: {j}") incr_j = CEL_ExprOp(f"{j} + 1").after(print_op).output with CEL_Condition(f"{incr_j} < {until_b}"): recur_b(incr_j, until_b) start_b = CEL_ExprOp("0").output with CEL_Condition(f"{start_b} < {until_b}"): b = recur_b(start_b, until_b) incr_i = CEL_ExprOp(f"{i} + 1").after(b).output with CEL_Condition(f"{incr_i} < {until_a}"): recur_a(incr_i, until_a) start_a = CEL_ExprOp("0").output with CEL_Condition(f"{start_a} < {until_a}"): recur_a(start_a, until_a) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler Compiler().compile(double_recursion_test, __file__.replace('.py', '.yaml'))
TektonCompiler._get_unique_id_code = Coder.empty class PrintOp(dsl.ContainerOp): def __init__(self, name: str, msg: str): super(PrintOp, self).__init__( name=name, image='alpine:3.6', command=['echo', msg, ">", "/tmp/stdout"], file_outputs={"stdout": "/tmp/stdout"} ) @dsl.graph_component def recur(i: int): decr_i = CEL_ConditionOp(f"{i} - 1").output PrintOp("print-iter", f"Iter: {decr_i}") with dsl.Condition(decr_i != 0): recur(decr_i) @dsl.pipeline("recur-and-condition") def recur_and_condition(iter_num: int = 42): recur(iter_num) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler Compiler().compile(recur_and_condition, __file__.replace('.py', '.yaml'))
task = dsl.ContainerOp(name=custom_task_name, image=custom_task_image, command=[custom_task_command], arguments=[ "--apiVersion", custom_task_api_version, "--kind", custom_task_kind, "--name", custom_task_resource_name, *custom_task_args(foo, bar, pi), resource_label, custom_task_resource ], file_outputs={ f"{result}": f"/{result}" for result in custom_task_results }) task.add_pod_annotation("valid_container", "false") return task def main_task_ref(foo: str = "Foo", bar="buzz", pi: int = 3.14): custom_task("--taskRef", foo, bar, pi) def main_task_spec(foo: str = "Foo", bar="buzz", pi: int = 3.14): custom_task("--taskSpec", foo, bar, pi) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler Compiler().compile(main_task_ref, __file__.replace('.py', '_ref.yaml')) Compiler().compile(main_task_spec, __file__.replace('.py', '_spec.yaml'))
class PrintOp(dsl.ContainerOp): def __init__(self, name: str, msg: str): super(PrintOp, self).__init__(name=name, image='alpine:3.6', command=['echo', msg, ">", "/tmp/stdout"], file_outputs={"stdout": "/tmp/stdout"}) class CelCondition(dsl.Condition): def __init__(self, pred: str, name: str = None): super().__init__(CEL_ConditionOp(pred).output == 'true', name) @dsl.graph_component def recur(i: int): decr_i = CEL_ConditionOp(f"{i} - 1").output PrintOp("print-iter", f"Iter: {decr_i}") recur(decr_i) @dsl.pipeline("condition-and-recur") def condition_and_recur(iter_num: int = 42): with CelCondition(f"{iter_num} != 0"): recur(iter_num) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler Compiler().compile(condition_and_recur, __file__.replace('.py', '.yaml'))
'--apiVersion', 'custom.tekton.dev/v1alpha1', '--kind', 'CelExprs', '--name', 'cel_exprs', '--ab', {inputValue: ab}, '--bc', {inputValue: bc}, {outputPath: ab}, {outputPath: bc} ] ''' % (CEL_EXPRS_IMAGE) ConditionOp_template = components.load_component_from_text(ConditionOp_yaml) conds_values = list(conds.values()) op = ConditionOp_template(conds_values[0], conds_values[1]) op.add_pod_annotation("valid_container", "false") return op @dsl.pipeline("nested-condition-test") def nested_condition_test(a: int, b: int, c: int): op = CEL_Exprs( ab=f"{a} < {b}", bc=f"{b} < {c}", ) with dsl.Condition(op.outputs["ab"] == 'true'): with dsl.Condition(op.outputs["bc"] == 'true'): print_op = PrintOp(f"{a} < {b} < {c}") if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler as Compiler Compiler().compile(nested_condition_test, __file__.replace('.py', '.yaml'))