def consume(param1): print(param1) @kfp.dsl.pipeline() def parallelfor_item_argument_resolving(): produce_str_task = produce_str() produce_list_of_strings_task = produce_list_of_strings() produce_list_of_ints_task = produce_list_of_ints() produce_list_of_dicts_task = produce_list_of_dicts() with kfp.dsl.ParallelFor(produce_list_of_strings_task.output) as loop_item: consume(produce_list_of_strings_task.output) consume(loop_item) consume(produce_str_task.output) with kfp.dsl.ParallelFor(produce_list_of_ints_task.output) as loop_item: consume(produce_list_of_ints_task.output) consume(loop_item) with kfp.dsl.ParallelFor(produce_list_of_dicts_task.output) as loop_item: consume(produce_list_of_dicts_task.output) # consume(loop_item) # Cannot use the full loop item when it's a dict consume(loop_item.aaa) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(parallelfor_item_argument_resolving, __file__.replace('.py', '.yaml'))
@dsl.pipeline(name='my-pipeline') def pipeline(): op0 = dsl.ContainerOp( name="my-out-cop0", image='python:alpine3.6', command=["sh", "-c"], arguments=[ 'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'], file_outputs={'out': '/tmp/out.json'}, ) with dsl.ParallelFor(op0.output) as item: op1 = dsl.ContainerOp( name="my-in-cop1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo do output op1 item: %s" % item], ) op_out = dsl.ContainerOp( name="my-out-cop2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo do output op2, outp: %s" % op0.output], ) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(pipeline, __file__.replace('.py', '.yaml'))
def __init__(self, name, msg): super(PrintOp, self).__init__( name=name, image='alpine:3.6', command=['echo', msg]) @dsl.pipeline( name='Flip Coin with Dependency', description='Shows how to use dsl.Condition.' ) def flipcoin(forced_result1: str = 'heads', forced_result2: str = 'tails'): flip = FlipCoinOp('flip', str(forced_result1)) with dsl.Condition(flip.output == 'heads') as condition: flip2 = FlipCoinOp('flip-again', str(forced_result2)) with dsl.Condition(flip2.output == 'tails'): PrintOp('print1', flip2.output) with dsl.Condition(flip.output == 'tails') as condition_2: PrintOp('print2', flip.output) PrintOp('print3', 'done').after(condition).after(condition_2) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(flipcoin, __file__.replace('.py', '.yaml'))
exit_op = ExitHandlerOp('exiting') with dsl.ExitHandler(exit_op): counter = GetFrequentWordOp( name='get-Frequent', message=message_param) counter.set_memory_request('200M') saver = SaveMessageOp( name='save', message=counter.output, output_path=output_path_param) saver.set_cpu_limit('0.5') saver.set_gpu_limit('2') saver.add_node_selector_constraint( 'cloud.google.com/gke-accelerator', 'nvidia-tesla-k80') saver.apply( gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12')) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler tkc = TektonCompiler() compiled_workflow = tkc._create_workflow( save_most_frequent_word, 'Save Most Frequent', 'Get Most Frequent Word and Save to GCS', [message_param, output_path_param], None) tkc._write_workflow(compiled_workflow, __file__.replace('.py', '.yaml'))
from kfp import onprem notebook_op.container.add_env_variable(V1EnvVar(name='DATA_DIR', value="${mount_path}")) notebook_op.apply(onprem.mount_pvc(pvc_name='${dataset_pvc}', volume_name='${dataset_pvc}', volume_mount_path='${mount_path}')) ############################################################ # Compile the pipeline ############################################################ pipeline_function = notebook_pipeline pipeline_filename = path.join(gettempdir(), pipeline_function.__name__ + '.pipeline.yaml') TektonCompiler().compile(pipeline_function, pipeline_filename) ############################################################ # Run the pipeline ############################################################ client = TektonClient(${pipeline_server}) # Get or create an experiment and submit a pipeline run experiment = client.create_experiment('NOTEBOOK_RUNS') # Submit the experiment to run in a pipeline run_name = '${run_name}' run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename)
for i in list( range(10) ): #try to be as close as possible to canonical python for DS people retcode_placeholder = gen_random_op(i).after(retcode_placeholder) #retcode_placeholder = print_gen_val_op(retcode_placeholder.output) retcode_placeholder = train_model_op(i, retcode_placeholder.output) print_gen_val_op(10).after(retcode_placeholder) #------------------------------------------------------ #kfp.compiler.Compiler().compile(run_pipeline, 'test_nested.yaml') from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(run_pipeline, 'test_sequential_param.yaml') ''' Q: can ops be wrapped in other ops op - deferred till pipeline executed non-op - run now https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html BaseOp Condition ContainerOp ExitHandler InputArgumentPath ParallelFor PipelineConf
notebook_op.container.add_env_variable( V1EnvVar(name='AWS_ACCESS_KEY_ID', value=cos_username)) notebook_op.container.add_env_variable( V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=cos_password)) notebook_op.container.set_image_pull_policy('Always') return notebook_op @dsl.pipeline( name='Sample kfp notebook pipeline', description='Sample template to compile and export a kfp notebook') def demo_pipeline(): run_notebook_op('dummy_name') # Compile the new pipeline TektonCompiler().compile(demo_pipeline, 'pipeline.tar.gz') # Upload the compiled pipeline client = kfp.Client(host=kfp_url) pipeline_info = client.upload_pipeline('pipeline.tar.gz', pipeline_name='pipeline-demo') # Create a new experiment experiment = client.create_experiment(name='demo-experiment') # Create a new run associated with experiment and our uploaded pipeline run = client.run_pipeline(experiment.id, 'demo-run', pipeline_id=pipeline_info.id)
} ) def echo2_op(text1): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "Text 1: $0";', text1] ) @dsl.pipeline( name='Hidden output file pipeline', description='Run a script that passes file to a non configurable path' ) def hidden_output_file_pipeline( ): """A three-step pipeline with the first two steps running in parallel.""" write_text = write_text_op() echo_task = echo2_op(write_text.output) if __name__ == '__main__': # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(hidden_output_file_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
def some_op(): return dsl.ContainerOp( name='sleep', image='busybox', command=['sleep 1'], ) @dsl.pipeline(name='affinity', description='A pipeline with affinity') def affinity_pipeline(): """A pipeline with affinity""" affinity = V1Affinity(node_affinity=V1NodeAffinity( required_during_scheduling_ignored_during_execution=V1NodeSelector( node_selector_terms=[ V1NodeSelectorTerm(match_expressions=[ V1NodeSelectorRequirement( key='beta.kubernetes.io/instance-type', operator='In', values=['p2.xlarge']) ]) ]))) some_op().add_affinity(affinity) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(affinity_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
fairness_check = fairness_check_ops(model_id='training-example', model_class_file=model_class_file, model_class_name=model_class_name, feature_testset_path=feature_testset_path, label_testset_path=label_testset_path, protected_label_testset_path=protected_label_testset_path, favorable_label=favorable_label, unfavorable_label=unfavorable_label, privileged_groups=privileged_groups, unprivileged_groups=unprivileged_groups, data_bucket_name='mlpipeline', result_bucket_name='mlpipeline').after(train_step).set_image_pull_policy("Always") robustness_check = robustness_check_ops(model_id='training-example', epsilon=fgsm_attack_epsilon, model_class_file=model_class_file, model_class_name=model_class_name, feature_testset_path=feature_testset_path, label_testset_path=label_testset_path, loss_fn=loss_fn, optimizer=optimizer, clip_values=clip_values, nb_classes=nb_classes, input_shape=input_shape, data_bucket_name='mlpipeline', result_bucket_name='mlpipeline').after(train_step).set_image_pull_policy("Always") if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(trusted_ai, __file__.replace('.py', '.yaml'))
num = random.randint(0, 9) return num # print the result @func_to_container_op def print_params(numbers_parm: int): print("The result number is: %d" % numbers_parm) # Pipeline to gen and echo the result def params_pipeline(): gen_task = gen_params() print_params(gen_task.output) # Combining all pipelines together in a single pipeline def file_passing_pipelines(): print_repeating_lines_pipeline() text_splitting_pipeline() sum_pipeline() params_pipeline() # General by kfp-tekton if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(file_passing_pipelines, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
- -c args: - | echo "Text 1: $0"; echo "Text 2: $1"; echo "{{inputs.parameters.gcs-download-data}}";\ echo "{{workflow.name}}"; echo "{{workflow.namespace}}"; echo "{{workflow.uid}}" - {inputValue: input1} - {inputValue: input2} """)(input1=text1, input2=text2) @dsl.pipeline( name='parallel-pipeline-with-argo-vars', description= 'Download two messages in parallel and prints the concatenated result and use Argo variables.' ) def download_and_join_with_argo_vars( url1: str = 'gs://ml-pipeline-playground/shakespeare1.txt', url2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'): """A three-step pipeline with the first two steps running in parallel with Argo variables.""" download1_task = gcs_download_op(url1) download2_task = gcs_download_op(url2) echo_task = echo2_op(download1_task.output, download2_task.output) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(download_and_join_with_argo_vars, __file__.replace('.py', '.yaml'))
# configures artifact location pipeline_artifact_location = dsl.ArtifactLocation.s3( bucket=bucket, endpoint="minio-service.%s:9000" % namespace, # parameterize minio-service endpoint insecure=True, access_key_secret=V1SecretKeySelector(name=secret_name, key="accesskey"), secret_key_secret={ "name": secret_name, "key": "secretkey" }, # accepts dict also ) # set pipeline level artifact location dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location) # artifacts in this op are stored to endpoint `minio-service.<namespace>:9000` op = dsl.ContainerOp(name="generate-output", image="busybox:%s" % tag, command=['sh', '-c', 'echo hello > /tmp/output.txt'], file_outputs={'output': '/tmp/output.txt'}) if __name__ == '__main__': # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(custom_artifact_location, __file__.replace('.py', '.yaml'), enable_artifacts=True)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kubernetes.client import V1Toleration from kfp.dsl import ContainerOp from kfp import dsl @dsl.pipeline(name='tolerations', description='A pipeline with tolerations') def tolerations(): """A pipeline with tolerations""" op1 = dsl.ContainerOp( name='download', image='busybox', command=['sh', '-c'], arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'], file_outputs={'downloaded': '/tmp/results.txt'})\ .add_toleration(V1Toleration(effect='NoSchedule', key='gpu', operator='Equal', value='run')) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(tolerations, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
def echo_op(text): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "$0"', text] ) @dsl.pipeline( name='Sequential pipeline', description='A pipeline with two sequential steps.' ) def sequential_pipeline( url='gs://ml-pipeline-playground/shakespeare1.txt', path='/tmp/results.txt' ): """A pipeline with two sequential steps.""" download_task = gcs_download_op(url) echo_task = echo_op(path) echo_task.after(download_task) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(sequential_pipeline, __file__.replace('.py', '.yaml'))
def notify_success(): print('SUCCESS!') @components.func_to_container_op def notify_failure(): print('FAILED!') @components.func_to_container_op def produce_number() -> int: import random rn = random.randrange(0, 1000) print(rn) return rn @dsl.pipeline(name='Conditions with global params') def conditions_with_global_params(n1='5', threshold='10', lower_bound='15'): add_numbers_task = add_numbers(n1, lower_bound) print_number_task = print_number(add_numbers_task.output) with dsl.Condition(print_number_task.output > threshold): notify_success() with dsl.Condition(print_number_task.output <= threshold): notify_failure() if __name__ == '__main__': TektonCompiler().compile(conditions_with_global_params, __file__.replace('.py', '.yaml'))
def echo_op(): return components.load_component_from_text(""" name: echo description: echo implementation: container: image: busybox command: - sh - -c args: - echo - Found my node """)() @dsl.pipeline(name='node-selector', description='A pipeline with Node Selector') def node_selector_pipeline(): """A pipeline with Node Selector""" echo_op().add_node_selector_constraint(label_name='kubernetes.io/os', value='linux') if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(node_selector_pipeline, __file__.replace('.py', '.yaml'))
container: image: library/bash:4.4.23 command: - sh - -c args: - echo foo > /mnt/file1 """ cop_op = components.load_component_from_text(COP_STR) @dsl.pipeline(name="volumeop-basic", description="A Basic Example on VolumeOp Usage.") def volumeop_basic(size: str = "10M"): vop = dsl.VolumeOp( name="create-pvc", resource_name="my-pvc", modes=dsl.VOLUME_MODE_RWO, size=size # success_condition="status.phase = Bound", # failure_condition="status.phase = Failed" ) cop = cop_op().add_pvolumes({"/mnt": vop.volume}) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(volumeop_basic, __file__.replace('.py', '.yaml'))
- {inputPath: text} """) def component_with_hardcoded_input_artifact_value(): """A component that passes hard-coded text as input artifact""" return component_with_input_artifact('hard-coded artifact value') def component_with_input_artifact_value_from_file(file_path): """A component that passes contents of a file as input artifact""" return component_with_input_artifact(Path(file_path).read_text()) @dsl.pipeline( name='pipeline-with-artifact-input-raw-argument-value', description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.' ) def input_artifact_pipeline(): component_with_inline_input_artifact('Constant artifact value') component_with_input_artifact('Constant artifact value') component_with_hardcoded_input_artifact_value() file_path = str(Path(__file__).parent.joinpath('input_artifact_raw_value.txt')) component_with_input_artifact_value_from_file(file_path) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(input_artifact_pipeline, __file__.replace('.py', '.yaml'))
description='email pipeline' ) def email_pipeline( server_secret="server-secret", subject="Hi, again!", body="Tekton email", sender="*****@*****.**", recipients="[email protected], [email protected]", attachment_filepath="/tmp/data/output.txt" ): email = email_op(server_secret=server_secret, subject=subject, body=body, sender=sender, recipients=recipients, attachment_path=attachment_filepath) email.add_env_variable(env_from_secret('USER', '$(params.server_secret)', 'user')) email.add_env_variable(env_from_secret('PASSWORD', '$(params.server_secret)', 'password')) email.add_env_variable(env_from_secret('TLS', '$(params.server_secret)', 'tls')) email.add_env_variable(env_from_secret('SERVER', '$(params.server_secret)', 'url')) email.add_env_variable(env_from_secret('PORT', '$(params.server_secret)', 'port')) email.apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path)) with dsl.ExitHandler(email): write_file_task = write_file(attachment_filepath).apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path)) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(email_pipeline, 'email_pipeline.yaml')
arguments=[ 'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' 'else \'tails\'; print(result)" | tee /tmp/output_result' ], file_outputs={'output_result': '/tmp/output_result'}) def print_op(msg): """Print a message.""" return dsl.ContainerOp( name='Print', image='alpine:3.6', command=['echo', msg], ) @dsl.pipeline(name='Tekton custom task on Kubeflow Pipeline', description='Shows how to use Tekton custom task with KFP') def custom_task_pipeline(): flip = flip_coin_op() flip2 = flip_coin_op() cel_condition = CEL_ConditionOp("'%s' == '%s'" % (flip.output, flip2.output)) print_op('Condition output is %s' % cel_condition.output) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(custom_task_pipeline, __file__.replace('.py', '.yaml'))
def echo_op(text): return dsl.ContainerOp(name='echo', image='library/bash:4', command=['sh', '-c'], arguments=['echo "$0"', text]) with open("dax_example.yaml", 'r') as stream: dax_example = yaml.safe_load(stream) # https://developer.ibm.com/components/data-asset-exchange/data/ # https://github.com/CODAIT/exchange-metadata-converter/blob/main/templates/mlx_out.yaml # https://github.com/IBM/dataset-lifecycle-framework @dsl.pipeline( name='dax-to-dlf', description= 'Convert a Data Asset EXchange YAML file to Data Lifecycle Framework YAML file.' ) def dax_to_dlf_pipeline(dax_yaml=yaml.safe_dump(dax_example)): dax_to_dlf = dax_to_dlf_op( dax_yaml=dax_yaml) # .set_image_pull_policy('Always') echo_op(dax_to_dlf.outputs["dlf_yaml"]) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(dax_to_dlf_pipeline, 'dax_to_dlf_pipeline.yaml')
command=['sh', '-c'], arguments=[ 'python -c "from collections import Counter; ' 'text = \'%s\'; ' 'print(\'Input: \' + text); ' 'words = Counter(text.split()); ' 'print(\'Most frequent word: \' + str(max(words, key=words.get)))" ' '| tee /tmp/message.txt' % message ], file_outputs={'word': '/tmp/message.txt'}) @dsl.pipeline(name='Save Most Frequent', description='Get Most Frequent Word and Save to GCS') # def save_most_frequent_word(message: str): def imagepullsecrets_pipeline( message="When flies fly behind flies, then flies are following flies." ): """A pipeline function describing the orchestration of the workflow.""" counter = GetFrequentWordOp(name='get-Frequent', message=message) # Call set_image_pull_secrets after get_pipeline_conf(). dsl.get_pipeline_conf() \ .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")]) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(imagepullsecrets_pipeline, __file__.replace('.py', '.yaml'))
step2_snap = dsl.VolumeSnapshotOp(name="step2_snap", resource_name="step2_snap", volume=step2.pvolume) step3 = dsl.ContainerOp(name="step3_copy", image="library/bash:4.4.23", command=["sh", "-c"], arguments=[ "mkdir /data/step3 && " "cp -av /data/step2/file1 /data/step3/file3" ], pvolumes={"/data": step2.pvolume}) step3_snap = dsl.VolumeSnapshotOp(name="step3_snap", resource_name="step3_snap", volume=step3.pvolume) step4 = dsl.ContainerOp( name="step4_output", image="library/bash:4.4.23", command=["cat", "/data/step2/file1", "/data/step3/file3"], pvolumes={"/data": step3.pvolume}) if __name__ == '__main__': # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(volume_snapshotop_sequential, __file__.replace('.py', '.yaml'))
import os os.makedirs(os.path.join(output_dir_path, 'subdir'), exist_ok=True) for i in range(num_files): file_path = os.path.join(output_dir_path, 'subdir', str(i) + '.txt') with open(file_path, 'w') as f: f.write(str(i)) produce_dir_python_task = produce_dir_with_files_python_op(num_files=15) get_subdir_task = get_subdir_op( # Input name "Input 1" is converted to pythonic parameter name "input_1" directory=produce_dir_python_task.output, subpath="subdir") list_items_task_1 = list_item_op_1( # Input name "Input 1" is converted to pythonic parameter name "input_1" directory=get_subdir_task.output) list_items_task_2 = list_item_op_2( # Input name "Input 1" is converted to pythonic parameter name "input_1" directory1=get_subdir_task.output, directory2=produce_dir_python_task.output) # General by kfp-tekton if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(create_component_pipeline, __file__.replace('.py', '.yaml'))
""" super(GetFrequentWordOp, self).__init__( name=name, image='python:3.5-jessie', command=['sh', '-c'], arguments=[ 'python -c "from collections import Counter; ' 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' '| tee /tmp/message.txt' % message ], file_outputs={'word': '/tmp/message.txt'}) @dsl.pipeline(name='Save Most Frequent', description='Get Most Frequent Word and Save to GCS') # def save_most_frequent_word(message: str): def imagepullsecrets_pipeline(message="This is a test"): """A pipeline function describing the orchestration of the workflow.""" counter = GetFrequentWordOp(name='get-Frequent', message=message) # Call set_image_pull_secrets after get_pipeline_conf(). dsl.get_pipeline_conf() \ .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")]) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(imagepullsecrets_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
command: - sh - -c - echo "$0" - {inputValue: text} """ echo_op = components.load_component_from_text(ECHO_STR) @dsl.pipeline( name='exit-handler', description= 'Downloads a message and prints it. The exit handler will run after the pipeline finishes (successfully or not).' ) def download_and_print( url: str = 'gs://ml-pipeline-playground/shakespeare1.txt'): """A sample pipeline showing exit handler.""" exit_task = echo_op('exit!') with dsl.ExitHandler(exit_task): download_task = gcs_download_op(url) echo_task = echo_op(download_task.outputs['data']) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(download_and_print, __file__.replace('.py', '.yaml'))
arguments=["sleep 15"]) task2 = dsl.ContainerOp( name="task2", image="registry.access.redhat.com/ubi8/ubi-minimal", command=["/bin/bash", "-c"], arguments=["sleep 200"]) task3 = dsl.ContainerOp( name="task3", image="registry.access.redhat.com/ubi8/ubi-minimal", command=["/bin/bash", "-c"], arguments=["sleep 300"]) flip_out = flip_coin_op() flip_out.after(task1) dsl.ContainerOp( name="task4", image="registry.access.redhat.com/ubi8/ubi-minimal", command=["/bin/bash", "-c"], arguments=["sleep 30"]).apply( after_any([task2, task3, flip_out.outputs['output'] == "heads"], "any_test", '/tekton/results/status')) if __name__ == "__main__": TektonCompiler().compile(any_sequence_pipeline, __file__.replace('.py', '.yaml'))
name: random-failure description: random failure inputs: - {name: exitcode, type: String} implementation: container: image: python:alpine3.6 command: - python - -c args: - | import random; import sys; exit_code = random.choice([int(i) for i in sys.argv[1].split(",")]); \ print(exit_code); sys.exit(exit_code) - {inputValue: exitcode} """)(exitcode=exit_codes) @dsl.pipeline( name='retry-random-failures', description='The pipeline includes two steps which fail randomly. It shows how to use ContainerOp(...).set_retry(...).' ) def retry_sample_pipeline(): op1 = random_failure_op('0,1,2,3').set_retry(10) op2 = random_failure_op('0,1').set_retry(5) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(retry_sample_pipeline, __file__.replace('.py', '.yaml'))
"message": "My name is %s" % fname1 }, { "first_name": fname2, "message": "My name is %s" % fname2 }] list_of_complex_dict = [{ "first_name": fname1, "message": produce_message(fname1).output }, { "first_name": fname2, "message": produce_message(fname2).output }] with kfp.dsl.ParallelFor(simple_list) as loop_item: consume(loop_item) with kfp.dsl.ParallelFor(list_of_dict) as loop_item2: consume(loop_item2.first_name) consume(loop_item2.message) with kfp.dsl.ParallelFor(list_of_complex_dict) as loop_item: consume(loop_item.first_name) consume(loop_item.message) if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(parallelfor_pipeline_param_in_items_resolving, __file__.replace('.py', '.yaml'))