def pipeline(): op0 = dsl.ContainerOp( name="my-out-cop0", image='python:alpine3.6', command=["sh", "-c"], arguments=[ 'python -c "import json; import sys; json.dump([{\'a\': 1, \'b\': 2}, {\'a\': 10, \'b\': 20}], open(\'/tmp/out.json\', \'w\'))"' ], file_outputs={'out': '/tmp/out.json'}, ) with dsl.ParallelFor(op0.output) as item: op1 = dsl.ContainerOp( name="my-in-cop1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo do output op1 item.a: %s" % item.a], ) op_out = dsl.ContainerOp( name="my-out-cop2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo do output op2, outp: %s" % op0.output], )
def pipeline(my_pipe_param: int = 10): loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}] with dsl.ParallelFor(loop_args) as item: op1 = dsl.ContainerOp( name="my-in-coop1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo op1 %s %s" % (item.a, my_pipe_param)], ) with dsl.ParallelFor([100, 200, 300]) as inner_item: op11 = dsl.ContainerOp( name="my-inner-inner-coop", image="library/bash:4.4.23", command=["sh", "-c"], arguments=[ "echo op1 %s %s %s" % (item.a, inner_item, my_pipe_param) ], ) op2 = dsl.ContainerOp( name="my-in-coop2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo op2 %s" % item.b], ) op_out = dsl.ContainerOp( name="my-out-cop", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo %s" % my_pipe_param], )
def volumeop_parallel(): vop = dsl.VolumeOp( name="create_pvc", resource_name="my-pvc", size="10Gi", modes=dsl.VOLUME_MODE_RWM) step1 = dsl.ContainerOp( name="step1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 1 | tee /mnt/file1"], pvolumes={"/mnt": vop.volume}) step2 = dsl.ContainerOp( name="step2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 2 | tee /common/file2"], pvolumes={"/common": vop.volume}) step3 = dsl.ContainerOp( name="step3", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 3 | tee /mnt3/file3"], pvolumes={"/mnt3": vop.volume})
def volumeop_sequential(): vop = dsl.VolumeOp( name="mypvc", resource_name="newpvc", size="10Gi", modes=dsl.VOLUME_MODE_RWM) step1 = dsl.ContainerOp( name="step1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 1|tee /data/file1"], pvolumes={"/data": vop.volume}) step2 = dsl.ContainerOp( name="step2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["cp /data/file1 /data/file2"], pvolumes={"/data": step1.pvolume}) step3 = dsl.ContainerOp( name="step3", image="library/bash:4.4.23", command=["cat", "/mnt/file1", "/mnt/file2"], pvolumes={"/mnt": step2.pvolume})
def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]): op0 = dsl.ContainerOp( name="my-out-cop0", image='python:alpine3.6', command=["sh", "-c"], arguments=[ 'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"' ], file_outputs={'out': '/tmp/out.json'}, ) with dsl.ParallelFor(loopidy_doop) as item: op1 = dsl.ContainerOp( name="my-in-cop1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo no output global op1, item.a: %s" % item.a], ).after(op0) op_out = dsl.ContainerOp( name="my-out-cop2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo no output global op2, outp: %s" % op0.output], )
def volume_snapshotop_sequential(url): vop = dsl.VolumeOp( name="create_volume", resource_name="vol1", size="1Gi", modes=dsl.VOLUME_MODE_RWM ) step1 = dsl.ContainerOp( name="step1_ingest", image="google/cloud-sdk:279.0.0", command=["sh", "-c"], arguments=["mkdir /data/step1 && " "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], pvolumes={"/data": vop.volume} ) step1_snap = dsl.VolumeSnapshotOp( name="step1_snap", resource_name="step1_snap", volume=step1.pvolume ) step2 = dsl.ContainerOp( name="step2_gunzip", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["mkdir /data/step2 && " "gunzip /data/step1/file1.gz -c >/data/step2/file1"], pvolumes={"/data": step1.pvolume} ) step2_snap = dsl.VolumeSnapshotOp( name="step2_snap", resource_name="step2_snap", volume=step2.pvolume ) step3 = dsl.ContainerOp( name="step3_copy", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["mkdir /data/step3 && " "cp -av /data/step2/file1 /data/step3/file3"], pvolumes={"/data": step2.pvolume} ) step3_snap = dsl.VolumeSnapshotOp( name="step3_snap", resource_name="step3_snap", volume=step3.pvolume ) step4 = dsl.ContainerOp( name="step4_output", image="library/bash:4.4.23", command=["cat", "/data/step2/file1", "/data/step3/file3"], pvolumes={"/data": step3.pvolume} )
def echo2_op(text1, text2): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "Text 1: $0"; echo "Text 2: $1"', text1, text2] )
def print_op(msg): """Print a message.""" return dsl.ContainerOp( name='Print', image='alpine:3.6', command=['echo', msg], )
def param_substitutions(): vop = dsl.VolumeOp(name="create_volume", resource_name="data", size="1Gi") op = dsl.ContainerOp(name="cop", image="image", arguments=["--param", vop.output], pvolumes={"/mnt": vop.volume})
def echo_op(): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "hello world"'] )
def param_substitutions(param): dsl.get_pipeline_conf().op_transformers.append(add_common_labels(param)) op = dsl.ContainerOp( name="cop", image="image", )
def echo_op(text): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "$0"', text] )
def sidecar_pipeline(): echo = dsl.Sidecar(name='echo', image='hashicorp/http-echo', args=['-text="hello world"']) op1 = dsl.ContainerOp( name='download', image='busybox', command=['sh', '-c'], arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'], sidecars=[echo], file_outputs={'downloaded': '/tmp/results.txt'}) op2 = dsl.ContainerOp(name='echo', image='library/bash', command=['sh', '-c'], arguments=['echo %s' % op1.output])
def gcs_download_op(url): return dsl.ContainerOp( name='GCS - Download', image='google/cloud-sdk:279.0.0', command=['sh', '-c'], arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], file_outputs={ 'data': '/tmp/results.txt', } )
def flip_coin_op(): """Flip a coin and output heads or tails randomly.""" return dsl.ContainerOp( name='Flip coin', image='python:alpine3.6', command=['sh', '-c'], arguments=['python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' 'else \'tails\'; print(result)" | tee /tmp/output'], file_outputs={'output': '/tmp/output'} )
def component_with_inline_input_artifact(text: str): return dsl.ContainerOp( name='component_with_inline_input_artifact', image='alpine', command=[ 'cat', dsl.InputArgumentPath(text, path='/tmp/inputs/text/data', input='text') ], # path and input are optional )
def volume_snapshotop_rokurl(rok_url): vop1 = dsl.VolumeOp(name="create_volume_1", resource_name="vol1", size="1Gi", annotations={"rok/origin": rok_url}, modes=dsl.VOLUME_MODE_RWM) step1 = dsl.ContainerOp( name="step1_concat", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["cat /data/file*| gzip -c >/data/full.gz"], pvolumes={"/data": vop1.volume}) step1_snap = dsl.VolumeSnapshotOp(name="create_snapshot_1", resource_name="snap1", volume=step1.pvolume) vop2 = dsl.VolumeOp(name="create_volume_2", resource_name="vol2", data_source=step1_snap.snapshot, size=step1_snap.outputs["size"]) step2 = dsl.ContainerOp(name="step2_gunzip", image="library/bash:4.4.23", command=["gunzip", "-k", "/data/full.gz"], pvolumes={"/data": vop2.volume}) step2_snap = dsl.VolumeSnapshotOp(name="create_snapshot_2", resource_name="snap2", volume=step2.pvolume) vop3 = dsl.VolumeOp(name="create_volume_3", resource_name="vol3", data_source=step2_snap.snapshot, size=step2_snap.outputs["size"]) step3 = dsl.ContainerOp(name="step3_output", image="library/bash:4.4.23", command=["cat", "/data/full"], pvolumes={"/data": vop3.volume})
def volume_pipeline(): op1 = dsl.ContainerOp( name='download', image='google/cloud-sdk', command=['sh', '-c'], arguments=['ls | tee /tmp/results.txt'], file_outputs={'downloaded': '/tmp/results.txt'}) \ .add_volume(k8s_client.V1Volume(name='gcp-credentials', secret=k8s_client.V1SecretVolumeSource( secret_name='user-gcp-sa'))) \ .add_volume_mount(k8s_client.V1VolumeMount( mount_path='/secret/gcp-credentials', name='gcp-credentials')) \ .add_env_variable(k8s_client.V1EnvVar( name='GOOGLE_APPLICATION_CREDENTIALS', value='/secret/gcp-credentials/user-gcp-sa.json')) \ .add_env_variable(k8s_client.V1EnvVar(name='Foo', value='bar')) op2 = dsl.ContainerOp( name='echo', image='library/bash', command=['sh', '-c'], arguments=['echo %s' % op1.output])
def component_with_input_artifact(text): """A component that passes text as input artifact.""" return dsl.ContainerOp( name='component_with_input_artifact', artifact_argument_paths=[ dsl.InputArgumentPath(argument=text, path='/tmp/inputs/text/data', input='text'), # path and input are optional ], image='alpine', command=['cat', '/tmp/inputs/text/data'], )
def volumeop_basic(size): vop = dsl.VolumeOp( name="create_pvc", resource_name="my-pvc", modes=dsl.VOLUME_MODE_RWM, size=size) cop = dsl.ContainerOp( name="cop", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo foo > /mnt/file1"], pvolumes={"/mnt": vop.volume})
def volume_op_dag(): vop = dsl.VolumeOp(name="create_pvc", resource_name="my-pvc", size="10Gi", modes=dsl.VOLUME_MODE_RWM) step1 = dsl.ContainerOp(name="step1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 1 | tee /mnt/file1"], pvolumes={"/mnt": vop.volume}) step2 = dsl.ContainerOp(name="step2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 2 | tee /mnt2/file2"], pvolumes={"/mnt2": vop.volume}) step3 = dsl.ContainerOp(name="step3", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["cat /mnt/file1 /mnt/file2"], pvolumes={"/mnt": vop.volume.after(step1, step2)})
def pipeline_with_sidecar(): # sidecar with sevice that reply "hello world" to any GET request echo = dsl.Sidecar( name="echo", image="nginx:1.13", command=["nginx", "-g", "daemon off;"], ) # container op with sidecar op1 = dsl.ContainerOp( name="download", image="busybox:latest", command=["sh", "-c"], arguments=[ "until wget http://localhost:80 -O /tmp/results.txt; do sleep 5; done && cat /tmp/results.txt" ], sidecars=[echo], file_outputs={"downloaded": "/tmp/results.txt"}, )
def resourceop_basic(username, password): secret_resource = k8s_client.V1Secret( api_version="v1", kind="Secret", metadata=k8s_client.V1ObjectMeta(generate_name="my-secret-"), type="Opaque", data={ "username": username, "password": password }) rop = dsl.ResourceOp(name="create-my-secret", k8s_resource=secret_resource, attribute_outputs={"name": "{.metadata.name}"}) secret = k8s_client.V1Volume( name="my-secret", secret=k8s_client.V1SecretVolumeSource(secret_name=rop.output)) cop = dsl.ContainerOp(name="cop", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["ls /etc/secret-volume"], pvolumes={"/etc/secret-volume": secret})
def echo2_graph_component(text2): dsl.ContainerOp(name='echo2-task1', image='library/bash:4.4.23', command=['sh', '-c'], arguments=['echo "$0"', text2])
def my_pipeline(): v1dsl.ContainerOp(name='comp1', image='gcr.io/dummy', command=['python', 'main.py'])