def consume(param1):
print(param1)
@kfp.dsl.pipeline()
def parallelfor_item_argument_resolving():
produce_str_task = produce_str()
produce_list_of_strings_task = produce_list_of_strings()
produce_list_of_ints_task = produce_list_of_ints()
produce_list_of_dicts_task = produce_list_of_dicts()
with kfp.dsl.ParallelFor(produce_list_of_strings_task.output) as loop_item:
consume(produce_list_of_strings_task.output)
consume(loop_item)
consume(produce_str_task.output)
with kfp.dsl.ParallelFor(produce_list_of_ints_task.output) as loop_item:
consume(produce_list_of_ints_task.output)
consume(loop_item)
with kfp.dsl.ParallelFor(produce_list_of_dicts_task.output) as loop_item:
consume(produce_list_of_dicts_task.output)
# consume(loop_item) # Cannot use the full loop item when it's a dict
consume(loop_item.aaa)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(parallelfor_item_argument_resolving,
__file__.replace('.py', '.yaml'))
@dsl.pipeline(name='my-pipeline')
def pipeline():
op0 = dsl.ContainerOp(
name="my-out-cop0",
image='python:alpine3.6',
command=["sh", "-c"],
arguments=[
'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'],
file_outputs={'out': '/tmp/out.json'},
)
with dsl.ParallelFor(op0.output) as item:
op1 = dsl.ContainerOp(
name="my-in-cop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op1 item: %s" % item],
)
op_out = dsl.ContainerOp(
name="my-out-cop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op2, outp: %s" % op0.output],
)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(pipeline, __file__.replace('.py', '.yaml'))
def __init__(self, name, msg):
super(PrintOp, self).__init__(
name=name,
image='alpine:3.6',
command=['echo', msg])
@dsl.pipeline(
name='Flip Coin with Dependency',
description='Shows how to use dsl.Condition.'
)
def flipcoin(forced_result1: str = 'heads', forced_result2: str = 'tails'):
flip = FlipCoinOp('flip', str(forced_result1))
with dsl.Condition(flip.output == 'heads') as condition:
flip2 = FlipCoinOp('flip-again', str(forced_result2))
with dsl.Condition(flip2.output == 'tails'):
PrintOp('print1', flip2.output)
with dsl.Condition(flip.output == 'tails') as condition_2:
PrintOp('print2', flip.output)
PrintOp('print3', 'done').after(condition).after(condition_2)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(flipcoin, __file__.replace('.py', '.yaml'))
exit_op = ExitHandlerOp('exiting')
with dsl.ExitHandler(exit_op):
counter = GetFrequentWordOp(
name='get-Frequent',
message=message_param)
counter.set_memory_request('200M')
saver = SaveMessageOp(
name='save',
message=counter.output,
output_path=output_path_param)
saver.set_cpu_limit('0.5')
saver.set_gpu_limit('2')
saver.add_node_selector_constraint(
'cloud.google.com/gke-accelerator',
'nvidia-tesla-k80')
saver.apply(
gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12'))
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
tkc = TektonCompiler()
compiled_workflow = tkc._create_workflow(
save_most_frequent_word,
'Save Most Frequent',
'Get Most Frequent Word and Save to GCS',
[message_param, output_path_param],
None)
tkc._write_workflow(compiled_workflow, __file__.replace('.py', '.yaml'))
from kfp import onprem
notebook_op.container.add_env_variable(V1EnvVar(name='DATA_DIR', value="${mount_path}"))
notebook_op.apply(onprem.mount_pvc(pvc_name='${dataset_pvc}',
volume_name='${dataset_pvc}',
volume_mount_path='${mount_path}'))
############################################################
# Compile the pipeline
############################################################
pipeline_function = notebook_pipeline
pipeline_filename = path.join(gettempdir(),
pipeline_function.__name__ + '.pipeline.yaml')
TektonCompiler().compile(pipeline_function, pipeline_filename)
############################################################
# Run the pipeline
############################################################
client = TektonClient(${pipeline_server})
# Get or create an experiment and submit a pipeline run
experiment = client.create_experiment('NOTEBOOK_RUNS')
# Submit the experiment to run in a pipeline
run_name = '${run_name}'
run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename)
for i in list(
range(10)
): #try to be as close as possible to canonical python for DS people
retcode_placeholder = gen_random_op(i).after(retcode_placeholder)
#retcode_placeholder = print_gen_val_op(retcode_placeholder.output)
retcode_placeholder = train_model_op(i, retcode_placeholder.output)
print_gen_val_op(10).after(retcode_placeholder)
#------------------------------------------------------
#kfp.compiler.Compiler().compile(run_pipeline, 'test_nested.yaml')
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(run_pipeline, 'test_sequential_param.yaml')
'''
Q: can ops be wrapped in other ops
op - deferred till pipeline executed
non-op - run now
https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html
BaseOp
Condition
ContainerOp
ExitHandler
InputArgumentPath
ParallelFor
PipelineConf
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_ACCESS_KEY_ID', value=cos_username))
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=cos_password))
notebook_op.container.set_image_pull_policy('Always')
return notebook_op
@dsl.pipeline(
name='Sample kfp notebook pipeline',
description='Sample template to compile and export a kfp notebook')
def demo_pipeline():
run_notebook_op('dummy_name')
# Compile the new pipeline
TektonCompiler().compile(demo_pipeline, 'pipeline.tar.gz')
# Upload the compiled pipeline
client = kfp.Client(host=kfp_url)
pipeline_info = client.upload_pipeline('pipeline.tar.gz',
pipeline_name='pipeline-demo')
# Create a new experiment
experiment = client.create_experiment(name='demo-experiment')
# Create a new run associated with experiment and our uploaded pipeline
run = client.run_pipeline(experiment.id,
'demo-run',
pipeline_id=pipeline_info.id)
}
)
def echo2_op(text1):
return dsl.ContainerOp(
name='echo',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "Text 1: $0";', text1]
)
@dsl.pipeline(
name='Hidden output file pipeline',
description='Run a script that passes file to a non configurable path'
)
def hidden_output_file_pipeline(
):
"""A three-step pipeline with the first two steps running in parallel."""
write_text = write_text_op()
echo_task = echo2_op(write_text.output)
if __name__ == '__main__':
# don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(hidden_output_file_pipeline, __file__.replace('.py', '.yaml'), generate_pipelinerun=True)
def some_op():
return dsl.ContainerOp(
name='sleep',
image='busybox',
command=['sleep 1'],
)
@dsl.pipeline(name='affinity', description='A pipeline with affinity')
def affinity_pipeline():
"""A pipeline with affinity"""
affinity = V1Affinity(node_affinity=V1NodeAffinity(
required_during_scheduling_ignored_during_execution=V1NodeSelector(
node_selector_terms=[
V1NodeSelectorTerm(match_expressions=[
V1NodeSelectorRequirement(
key='beta.kubernetes.io/instance-type',
operator='In',
values=['p2.xlarge'])
])
])))
some_op().add_affinity(affinity)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(affinity_pipeline,
__file__.replace('.py', '.yaml'),
generate_pipelinerun=True)
fairness_check = fairness_check_ops(model_id='training-example',
model_class_file=model_class_file,
model_class_name=model_class_name,
feature_testset_path=feature_testset_path,
label_testset_path=label_testset_path,
protected_label_testset_path=protected_label_testset_path,
favorable_label=favorable_label,
unfavorable_label=unfavorable_label,
privileged_groups=privileged_groups,
unprivileged_groups=unprivileged_groups,
data_bucket_name='mlpipeline',
result_bucket_name='mlpipeline').after(train_step).set_image_pull_policy("Always")
robustness_check = robustness_check_ops(model_id='training-example',
epsilon=fgsm_attack_epsilon,
model_class_file=model_class_file,
model_class_name=model_class_name,
feature_testset_path=feature_testset_path,
label_testset_path=label_testset_path,
loss_fn=loss_fn,
optimizer=optimizer,
clip_values=clip_values,
nb_classes=nb_classes,
input_shape=input_shape,
data_bucket_name='mlpipeline',
result_bucket_name='mlpipeline').after(train_step).set_image_pull_policy("Always")
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(trusted_ai, __file__.replace('.py', '.yaml'))
num = random.randint(0, 9)
return num
# print the result
@func_to_container_op
def print_params(numbers_parm: int):
print("The result number is: %d" % numbers_parm)
# Pipeline to gen and echo the result
def params_pipeline():
gen_task = gen_params()
print_params(gen_task.output)
# Combining all pipelines together in a single pipeline
def file_passing_pipelines():
print_repeating_lines_pipeline()
text_splitting_pipeline()
sum_pipeline()
params_pipeline()
# General by kfp-tekton
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(file_passing_pipelines,
__file__.replace('.py', '.yaml'),
generate_pipelinerun=True)
- -c
args:
- |
echo "Text 1: $0"; echo "Text 2: $1"; echo "{{inputs.parameters.gcs-download-data}}";\
echo "{{workflow.name}}"; echo "{{workflow.namespace}}"; echo "{{workflow.uid}}"
- {inputValue: input1}
- {inputValue: input2}
""")(input1=text1, input2=text2)
@dsl.pipeline(
name='parallel-pipeline-with-argo-vars',
description=
'Download two messages in parallel and prints the concatenated result and use Argo variables.'
)
def download_and_join_with_argo_vars(
url1: str = 'gs://ml-pipeline-playground/shakespeare1.txt',
url2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'):
"""A three-step pipeline with the first two steps running in parallel with Argo variables."""
download1_task = gcs_download_op(url1)
download2_task = gcs_download_op(url2)
echo_task = echo2_op(download1_task.output, download2_task.output)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(download_and_join_with_argo_vars,
__file__.replace('.py', '.yaml'))
# configures artifact location
pipeline_artifact_location = dsl.ArtifactLocation.s3(
bucket=bucket,
endpoint="minio-service.%s:9000" %
namespace, # parameterize minio-service endpoint
insecure=True,
access_key_secret=V1SecretKeySelector(name=secret_name,
key="accesskey"),
secret_key_secret={
"name": secret_name,
"key": "secretkey"
}, # accepts dict also
)
# set pipeline level artifact location
dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location)
# artifacts in this op are stored to endpoint `minio-service.<namespace>:9000`
op = dsl.ContainerOp(name="generate-output",
image="busybox:%s" % tag,
command=['sh', '-c', 'echo hello > /tmp/output.txt'],
file_outputs={'output': '/tmp/output.txt'})
if __name__ == '__main__':
# don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(custom_artifact_location,
__file__.replace('.py', '.yaml'),
enable_artifacts=True)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kubernetes.client import V1Toleration
from kfp.dsl import ContainerOp
from kfp import dsl
@dsl.pipeline(name='tolerations', description='A pipeline with tolerations')
def tolerations():
"""A pipeline with tolerations"""
op1 = dsl.ContainerOp(
name='download',
image='busybox',
command=['sh', '-c'],
arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'],
file_outputs={'downloaded': '/tmp/results.txt'})\
.add_toleration(V1Toleration(effect='NoSchedule',
key='gpu',
operator='Equal',
value='run'))
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(tolerations,
__file__.replace('.py', '.yaml'),
generate_pipelinerun=True)
def echo_op(text):
return dsl.ContainerOp(
name='echo',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "$0"', text]
)
@dsl.pipeline(
name='Sequential pipeline',
description='A pipeline with two sequential steps.'
)
def sequential_pipeline(
url='gs://ml-pipeline-playground/shakespeare1.txt',
path='/tmp/results.txt'
):
"""A pipeline with two sequential steps."""
download_task = gcs_download_op(url)
echo_task = echo_op(path)
echo_task.after(download_task)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(sequential_pipeline, __file__.replace('.py', '.yaml'))
def notify_success():
print('SUCCESS!')
@components.func_to_container_op
def notify_failure():
print('FAILED!')
@components.func_to_container_op
def produce_number() -> int:
import random
rn = random.randrange(0, 1000)
print(rn)
return rn
@dsl.pipeline(name='Conditions with global params')
def conditions_with_global_params(n1='5', threshold='10', lower_bound='15'):
add_numbers_task = add_numbers(n1, lower_bound)
print_number_task = print_number(add_numbers_task.output)
with dsl.Condition(print_number_task.output > threshold):
notify_success()
with dsl.Condition(print_number_task.output <= threshold):
notify_failure()
if __name__ == '__main__':
TektonCompiler().compile(conditions_with_global_params, __file__.replace('.py', '.yaml'))
def echo_op():
return components.load_component_from_text("""
name: echo
description: echo
implementation:
container:
image: busybox
command:
- sh
- -c
args:
- echo
- Found my node
""")()
@dsl.pipeline(name='node-selector',
description='A pipeline with Node Selector')
def node_selector_pipeline():
"""A pipeline with Node Selector"""
echo_op().add_node_selector_constraint(label_name='kubernetes.io/os',
value='linux')
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(node_selector_pipeline,
__file__.replace('.py', '.yaml'))
container:
image: library/bash:4.4.23
command:
- sh
- -c
args:
- echo foo > /mnt/file1
"""
cop_op = components.load_component_from_text(COP_STR)
@dsl.pipeline(name="volumeop-basic",
description="A Basic Example on VolumeOp Usage.")
def volumeop_basic(size: str = "10M"):
vop = dsl.VolumeOp(
name="create-pvc",
resource_name="my-pvc",
modes=dsl.VOLUME_MODE_RWO,
size=size
# success_condition="status.phase = Bound",
# failure_condition="status.phase = Failed"
)
cop = cop_op().add_pvolumes({"/mnt": vop.volume})
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(volumeop_basic, __file__.replace('.py', '.yaml'))
- {inputPath: text}
""")
def component_with_hardcoded_input_artifact_value():
"""A component that passes hard-coded text as input artifact"""
return component_with_input_artifact('hard-coded artifact value')
def component_with_input_artifact_value_from_file(file_path):
"""A component that passes contents of a file as input artifact"""
return component_with_input_artifact(Path(file_path).read_text())
@dsl.pipeline(
name='pipeline-with-artifact-input-raw-argument-value',
description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.'
)
def input_artifact_pipeline():
component_with_inline_input_artifact('Constant artifact value')
component_with_input_artifact('Constant artifact value')
component_with_hardcoded_input_artifact_value()
file_path = str(Path(__file__).parent.joinpath('input_artifact_raw_value.txt'))
component_with_input_artifact_value_from_file(file_path)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(input_artifact_pipeline, __file__.replace('.py', '.yaml'))
description='email pipeline'
)
def email_pipeline(
server_secret="server-secret",
subject="Hi, again!",
body="Tekton email",
sender="*****@*****.**",
recipients="[email protected], [email protected]",
attachment_filepath="/tmp/data/output.txt"
):
email = email_op(server_secret=server_secret,
subject=subject,
body=body,
sender=sender,
recipients=recipients,
attachment_path=attachment_filepath)
email.add_env_variable(env_from_secret('USER', '$(params.server_secret)', 'user'))
email.add_env_variable(env_from_secret('PASSWORD', '$(params.server_secret)', 'password'))
email.add_env_variable(env_from_secret('TLS', '$(params.server_secret)', 'tls'))
email.add_env_variable(env_from_secret('SERVER', '$(params.server_secret)', 'url'))
email.add_env_variable(env_from_secret('PORT', '$(params.server_secret)', 'port'))
email.apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path))
with dsl.ExitHandler(email):
write_file_task = write_file(attachment_filepath).apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path))
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(email_pipeline, 'email_pipeline.yaml')
arguments=[
'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 '
'else \'tails\'; print(result)" | tee /tmp/output_result'
],
file_outputs={'output_result': '/tmp/output_result'})
def print_op(msg):
"""Print a message."""
return dsl.ContainerOp(
name='Print',
image='alpine:3.6',
command=['echo', msg],
)
@dsl.pipeline(name='Tekton custom task on Kubeflow Pipeline',
description='Shows how to use Tekton custom task with KFP')
def custom_task_pipeline():
flip = flip_coin_op()
flip2 = flip_coin_op()
cel_condition = CEL_ConditionOp("'%s' == '%s'" %
(flip.output, flip2.output))
print_op('Condition output is %s' % cel_condition.output)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(custom_task_pipeline,
__file__.replace('.py', '.yaml'))
def echo_op(text):
return dsl.ContainerOp(name='echo',
image='library/bash:4',
command=['sh', '-c'],
arguments=['echo "$0"', text])
with open("dax_example.yaml", 'r') as stream:
dax_example = yaml.safe_load(stream)
# https://developer.ibm.com/components/data-asset-exchange/data/
# https://github.com/CODAIT/exchange-metadata-converter/blob/main/templates/mlx_out.yaml
# https://github.com/IBM/dataset-lifecycle-framework
@dsl.pipeline(
name='dax-to-dlf',
description=
'Convert a Data Asset EXchange YAML file to Data Lifecycle Framework YAML file.'
)
def dax_to_dlf_pipeline(dax_yaml=yaml.safe_dump(dax_example)):
dax_to_dlf = dax_to_dlf_op(
dax_yaml=dax_yaml) # .set_image_pull_policy('Always')
echo_op(dax_to_dlf.outputs["dlf_yaml"])
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(dax_to_dlf_pipeline, 'dax_to_dlf_pipeline.yaml')
command=['sh', '-c'],
arguments=[
'python -c "from collections import Counter; '
'text = \'%s\'; '
'print(\'Input: \' + text); '
'words = Counter(text.split()); '
'print(\'Most frequent word: \' + str(max(words, key=words.get)))" '
'| tee /tmp/message.txt' % message
],
file_outputs={'word': '/tmp/message.txt'})
@dsl.pipeline(name='Save Most Frequent',
description='Get Most Frequent Word and Save to GCS')
# def save_most_frequent_word(message: str):
def imagepullsecrets_pipeline(
message="When flies fly behind flies, then flies are following flies."
):
"""A pipeline function describing the orchestration of the workflow."""
counter = GetFrequentWordOp(name='get-Frequent', message=message)
# Call set_image_pull_secrets after get_pipeline_conf().
dsl.get_pipeline_conf() \
.set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")])
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(imagepullsecrets_pipeline,
__file__.replace('.py', '.yaml'))
step2_snap = dsl.VolumeSnapshotOp(name="step2_snap",
resource_name="step2_snap",
volume=step2.pvolume)
step3 = dsl.ContainerOp(name="step3_copy",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=[
"mkdir /data/step3 && "
"cp -av /data/step2/file1 /data/step3/file3"
],
pvolumes={"/data": step2.pvolume})
step3_snap = dsl.VolumeSnapshotOp(name="step3_snap",
resource_name="step3_snap",
volume=step3.pvolume)
step4 = dsl.ContainerOp(
name="step4_output",
image="library/bash:4.4.23",
command=["cat", "/data/step2/file1", "/data/step3/file3"],
pvolumes={"/data": step3.pvolume})
if __name__ == '__main__':
# don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(volume_snapshotop_sequential,
__file__.replace('.py', '.yaml'))
import os
os.makedirs(os.path.join(output_dir_path, 'subdir'), exist_ok=True)
for i in range(num_files):
file_path = os.path.join(output_dir_path, 'subdir',
str(i) + '.txt')
with open(file_path, 'w') as f:
f.write(str(i))
produce_dir_python_task = produce_dir_with_files_python_op(num_files=15)
get_subdir_task = get_subdir_op(
# Input name "Input 1" is converted to pythonic parameter name "input_1"
directory=produce_dir_python_task.output,
subpath="subdir")
list_items_task_1 = list_item_op_1(
# Input name "Input 1" is converted to pythonic parameter name "input_1"
directory=get_subdir_task.output)
list_items_task_2 = list_item_op_2(
# Input name "Input 1" is converted to pythonic parameter name "input_1"
directory1=get_subdir_task.output,
directory2=produce_dir_python_task.output)
# General by kfp-tekton
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(create_component_pipeline,
__file__.replace('.py', '.yaml'))
"""
super(GetFrequentWordOp, self).__init__(
name=name,
image='python:3.5-jessie',
command=['sh', '-c'],
arguments=[
'python -c "from collections import Counter; '
'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" '
'| tee /tmp/message.txt' % message
],
file_outputs={'word': '/tmp/message.txt'})
@dsl.pipeline(name='Save Most Frequent',
description='Get Most Frequent Word and Save to GCS')
# def save_most_frequent_word(message: str):
def imagepullsecrets_pipeline(message="This is a test"):
"""A pipeline function describing the orchestration of the workflow."""
counter = GetFrequentWordOp(name='get-Frequent', message=message)
# Call set_image_pull_secrets after get_pipeline_conf().
dsl.get_pipeline_conf() \
.set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")])
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(imagepullsecrets_pipeline,
__file__.replace('.py', '.yaml'),
generate_pipelinerun=True)
command:
- sh
- -c
- echo "$0"
- {inputValue: text}
"""
echo_op = components.load_component_from_text(ECHO_STR)
@dsl.pipeline(
name='exit-handler',
description=
'Downloads a message and prints it. The exit handler will run after the pipeline finishes (successfully or not).'
)
def download_and_print(
url: str = 'gs://ml-pipeline-playground/shakespeare1.txt'):
"""A sample pipeline showing exit handler."""
exit_task = echo_op('exit!')
with dsl.ExitHandler(exit_task):
download_task = gcs_download_op(url)
echo_task = echo_op(download_task.outputs['data'])
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(download_and_print,
__file__.replace('.py', '.yaml'))
arguments=["sleep 15"])
task2 = dsl.ContainerOp(
name="task2",
image="registry.access.redhat.com/ubi8/ubi-minimal",
command=["/bin/bash", "-c"],
arguments=["sleep 200"])
task3 = dsl.ContainerOp(
name="task3",
image="registry.access.redhat.com/ubi8/ubi-minimal",
command=["/bin/bash", "-c"],
arguments=["sleep 300"])
flip_out = flip_coin_op()
flip_out.after(task1)
dsl.ContainerOp(
name="task4",
image="registry.access.redhat.com/ubi8/ubi-minimal",
command=["/bin/bash", "-c"],
arguments=["sleep 30"]).apply(
after_any([task2, task3, flip_out.outputs['output'] == "heads"],
"any_test", '/tekton/results/status'))
if __name__ == "__main__":
TektonCompiler().compile(any_sequence_pipeline,
__file__.replace('.py', '.yaml'))
name: random-failure
description: random failure
inputs:
- {name: exitcode, type: String}
implementation:
container:
image: python:alpine3.6
command:
- python
- -c
args:
- |
import random; import sys; exit_code = random.choice([int(i) for i in sys.argv[1].split(",")]); \
print(exit_code); sys.exit(exit_code)
- {inputValue: exitcode}
""")(exitcode=exit_codes)
@dsl.pipeline(
name='retry-random-failures',
description='The pipeline includes two steps which fail randomly. It shows how to use ContainerOp(...).set_retry(...).'
)
def retry_sample_pipeline():
op1 = random_failure_op('0,1,2,3').set_retry(10)
op2 = random_failure_op('0,1').set_retry(5)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(retry_sample_pipeline, __file__.replace('.py', '.yaml'))
"message": "My name is %s" % fname1
}, {
"first_name": fname2,
"message": "My name is %s" % fname2
}]
list_of_complex_dict = [{
"first_name": fname1,
"message": produce_message(fname1).output
}, {
"first_name": fname2,
"message": produce_message(fname2).output
}]
with kfp.dsl.ParallelFor(simple_list) as loop_item:
consume(loop_item)
with kfp.dsl.ParallelFor(list_of_dict) as loop_item2:
consume(loop_item2.first_name)
consume(loop_item2.message)
with kfp.dsl.ParallelFor(list_of_complex_dict) as loop_item:
consume(loop_item.first_name)
consume(loop_item.message)
if __name__ == '__main__':
from kfp_tekton.compiler import TektonCompiler
TektonCompiler().compile(parallelfor_pipeline_param_in_items_resolving,
__file__.replace('.py', '.yaml'))
