def main(fdir: Optional[str] = None) -> List[str]:
fpaths: List[str] = list()
from kfp_tekton.compiler import TektonCompiler as Compiler
max_length = max(lengths)
op_name_base = random_chars_of_len(max_length)
for length in lengths:
main_pipeline = pipeline_for_len(length, op_name_base)
fpath = __file__.replace('.py', f'-{length}.yaml')
fname = os.path.basename(fpath)
if fdir is not None:
fpath = os.path.join(fdir, fname)
if length > 57:
# Op name cannot be more than 57 characters
with pytest.raises(ValueError):
Compiler().compile(main_pipeline, fpath)
else:
Compiler().compile(main_pipeline, fpath)
fpaths.append(fpath)
return fpaths
image: library/bash:4.4.23
command: ['sh', '-c']
args:
- |
set -e
echo op1 "$0" "$1"
- {inputValue: item}
- {inputValue: param}
'''
@dsl.pipeline(name='pipeline')
def pipeline(param: int = 10):
loop_args = [1, 2]
with dsl.ParallelFor(loop_args, parallelism=1) as item:
op1_template = components.load_component_from_text(op1_yaml)
op1 = op1_template(item, param)
condi_1 = tekton.CEL_ConditionOp(f"{item} == 2").output
with dsl.Condition(condi_1 == 'true'):
tekton.Break()
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf
tekton_pipeline_conf = TektonPipelineConf()
tekton_pipeline_conf.set_tekton_inline_spec(True)
Compiler().compile(pipeline,
__file__.replace('.py', '.yaml'),
tekton_pipeline_conf=tekton_pipeline_conf)
set -e
echo $0 > $1
- {inputValue: input_text}
- {outputPath: output_value}
""" % (name))
return print_op(msg)
@dsl.graph_component
def function_the_name_of_which_is_exactly_51_chars_long(i: int):
decr_i = CEL_ConditionOp(f"{i} - 1").output
PrintOp("print-iter", f"Iter: {decr_i}")
with dsl.Condition(decr_i != 0):
function_the_name_of_which_is_exactly_51_chars_long(decr_i)
@dsl.pipeline("pipeline-the-name-of-which-is-exactly-51-chars-long")
def pipeline_the_name_of_which_is_exactly_51_chars_long(iter_num: int = 42):
function_the_name_of_which_is_exactly_51_chars_long(iter_num)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
from kfp_tekton.compiler.pipeline_utils import TektonPipelineConf
tekton_pipeline_conf = TektonPipelineConf()
tekton_pipeline_conf.set_tekton_inline_spec(False)
tekton_pipeline_conf.set_resource_in_separate_yaml(True)
Compiler().compile(pipeline_the_name_of_which_is_exactly_51_chars_long,
__file__.replace('.py', '.yaml'),
tekton_pipeline_conf=tekton_pipeline_conf)
@dsl.pipeline(
name='nested-recursion-pipeline',
description='shows how to use graph_component and nested recursion.')
def flipcoin(maxVal: int = 12):
@dsl._component.graph_component
def flip_component(flip_result, maxVal):
@dsl._component.graph_component
def flip_component_b(flip_result_b, maxVal_b):
with dsl.Condition(flip_result_b == 'heads'):
print_flip_b = print_op(flip_result_b)
flipB = flip_coin_op().after(print_flip_b)
flip_component_b(flipB.output, maxVal_b)
with dsl.Condition(flip_result == 'heads'):
flip_component_b(flip_result, maxVal)
with dsl.Condition(flip_result == 'tails'):
print_flip = print_op(flip_result)
flipA = flip_coin_op().after(print_flip)
flip_component(flipA.output, maxVal)
flip_out = flip_coin_op()
flip_loop = flip_component(flip_out.output, maxVal)
print_op('cool, it is over. %s' % flip_out.output).after(flip_loop)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
# For Argo, uncomment the line below
# from kfp.compiler import Compiler
Compiler().compile(flipcoin, __file__.replace('.py', '.yaml'))
def gcs_download_op(url):
return dsl.ContainerOp(
name='GCS - Download' * 4,
image='google/cloud-sdk:279.0.0',
command=['sh', '-c'],
arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'],
file_outputs={
'data' * 10: '/tmp/results.txt',
})
class PrintOp(dsl.ContainerOp):
def __init__(self, name, msg):
super(PrintOp, self).__init__(name=name,
image='alpine:3.6',
command=['echo', msg])
@dsl.pipeline(
name="Some very long name with lots of words in it. " +
"It should be over 63 chars long in order to observe the problem.")
def main_fn(url1='gs://ml-pipeline-playground/shakespeare1.txt'):
download1_task = gcs_download_op(url1)
PrintOp('print' * 10, download1_task.output)
if __name__ == '__main__':
Compiler().compile(main_fn, __file__.replace('.py', '.yaml'))
return CEL_ConditionOp(expr)
@dsl.pipeline("double-recursion-test")
def double_recursion_test(until_a: int = 4, until_b: int = 3):
@dsl.graph_component
def recur_a(i: int, until_a: int):
@dsl.graph_component
def recur_b(j: int, until_b: int):
print_op = PrintOp(f"Iter A: {i}, B: {j}")
incr_j = CEL_ExprOp(f"{j} + 1").after(print_op).output
with CEL_Condition(f"{incr_j} < {until_b}"):
recur_b(incr_j, until_b)
start_b = CEL_ExprOp("0").output
with CEL_Condition(f"{start_b} < {until_b}"):
b = recur_b(start_b, until_b)
incr_i = CEL_ExprOp(f"{i} + 1").after(b).output
with CEL_Condition(f"{incr_i} < {until_a}"):
recur_a(incr_i, until_a)
start_a = CEL_ExprOp("0").output
with CEL_Condition(f"{start_a} < {until_a}"):
recur_a(start_a, until_a)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
Compiler().compile(double_recursion_test, __file__.replace('.py', '.yaml'))
TektonCompiler._get_unique_id_code = Coder.empty
class PrintOp(dsl.ContainerOp):
def __init__(self, name: str, msg: str):
super(PrintOp, self).__init__(
name=name,
image='alpine:3.6',
command=['echo', msg, ">", "/tmp/stdout"],
file_outputs={"stdout": "/tmp/stdout"}
)
@dsl.graph_component
def recur(i: int):
decr_i = CEL_ConditionOp(f"{i} - 1").output
PrintOp("print-iter", f"Iter: {decr_i}")
with dsl.Condition(decr_i != 0):
recur(decr_i)
@dsl.pipeline("recur-and-condition")
def recur_and_condition(iter_num: int = 42):
recur(iter_num)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
Compiler().compile(recur_and_condition, __file__.replace('.py', '.yaml'))
task = dsl.ContainerOp(name=custom_task_name,
image=custom_task_image,
command=[custom_task_command],
arguments=[
"--apiVersion", custom_task_api_version,
"--kind", custom_task_kind, "--name",
custom_task_resource_name,
*custom_task_args(foo, bar, pi), resource_label,
custom_task_resource
],
file_outputs={
f"{result}": f"/{result}"
for result in custom_task_results
})
task.add_pod_annotation("valid_container", "false")
return task
def main_task_ref(foo: str = "Foo", bar="buzz", pi: int = 3.14):
custom_task("--taskRef", foo, bar, pi)
def main_task_spec(foo: str = "Foo", bar="buzz", pi: int = 3.14):
custom_task("--taskSpec", foo, bar, pi)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
Compiler().compile(main_task_ref, __file__.replace('.py', '_ref.yaml'))
Compiler().compile(main_task_spec, __file__.replace('.py', '_spec.yaml'))
class PrintOp(dsl.ContainerOp):
def __init__(self, name: str, msg: str):
super(PrintOp,
self).__init__(name=name,
image='alpine:3.6',
command=['echo', msg, ">", "/tmp/stdout"],
file_outputs={"stdout": "/tmp/stdout"})
class CelCondition(dsl.Condition):
def __init__(self, pred: str, name: str = None):
super().__init__(CEL_ConditionOp(pred).output == 'true', name)
@dsl.graph_component
def recur(i: int):
decr_i = CEL_ConditionOp(f"{i} - 1").output
PrintOp("print-iter", f"Iter: {decr_i}")
recur(decr_i)
@dsl.pipeline("condition-and-recur")
def condition_and_recur(iter_num: int = 42):
with CelCondition(f"{iter_num} != 0"):
recur(iter_num)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
Compiler().compile(condition_and_recur, __file__.replace('.py', '.yaml'))
'--apiVersion', 'custom.tekton.dev/v1alpha1',
'--kind', 'CelExprs',
'--name', 'cel_exprs',
'--ab', {inputValue: ab},
'--bc', {inputValue: bc},
{outputPath: ab},
{outputPath: bc}
]
''' % (CEL_EXPRS_IMAGE)
ConditionOp_template = components.load_component_from_text(ConditionOp_yaml)
conds_values = list(conds.values())
op = ConditionOp_template(conds_values[0], conds_values[1])
op.add_pod_annotation("valid_container", "false")
return op
@dsl.pipeline("nested-condition-test")
def nested_condition_test(a: int, b: int, c: int):
op = CEL_Exprs(
ab=f"{a} < {b}",
bc=f"{b} < {c}",
)
with dsl.Condition(op.outputs["ab"] == 'true'):
with dsl.Condition(op.outputs["bc"] == 'true'):
print_op = PrintOp(f"{a} < {b} < {c}")
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler as Compiler
Compiler().compile(nested_condition_test, __file__.replace('.py', '.yaml'))