def pipeline():
op0 = dsl.ContainerOp(
name="my-out-cop0",
image='python:alpine3.6',
command=["sh", "-c"],
arguments=[
'python -c "import json; import sys; json.dump([{\'a\': 1, \'b\': 2}, {\'a\': 10, \'b\': 20}], open(\'/tmp/out.json\', \'w\'))"'
],
file_outputs={'out': '/tmp/out.json'},
)
with dsl.ParallelFor(op0.output) as item:
op1 = dsl.ContainerOp(
name="my-in-cop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op1 item.a: %s" % item.a],
)
op_out = dsl.ContainerOp(
name="my-out-cop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op2, outp: %s" % op0.output],
)
def pipeline(my_pipe_param: int = 10):
loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]
with dsl.ParallelFor(loop_args) as item:
op1 = dsl.ContainerOp(
name="my-in-coop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op1 %s %s" % (item.a, my_pipe_param)],
)
with dsl.ParallelFor([100, 200, 300]) as inner_item:
op11 = dsl.ContainerOp(
name="my-inner-inner-coop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=[
"echo op1 %s %s %s" % (item.a, inner_item, my_pipe_param)
],
)
op2 = dsl.ContainerOp(
name="my-in-coop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op2 %s" % item.b],
)
op_out = dsl.ContainerOp(
name="my-out-cop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo %s" % my_pipe_param],
)
def volumeop_parallel():
vop = dsl.VolumeOp(
name="create_pvc",
resource_name="my-pvc",
size="10Gi",
modes=dsl.VOLUME_MODE_RWM)
step1 = dsl.ContainerOp(
name="step1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 1 | tee /mnt/file1"],
pvolumes={"/mnt": vop.volume})
step2 = dsl.ContainerOp(
name="step2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 2 | tee /common/file2"],
pvolumes={"/common": vop.volume})
step3 = dsl.ContainerOp(
name="step3",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 3 | tee /mnt3/file3"],
pvolumes={"/mnt3": vop.volume})
def volumeop_sequential():
vop = dsl.VolumeOp(
name="mypvc",
resource_name="newpvc",
size="10Gi",
modes=dsl.VOLUME_MODE_RWM)
step1 = dsl.ContainerOp(
name="step1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 1|tee /data/file1"],
pvolumes={"/data": vop.volume})
step2 = dsl.ContainerOp(
name="step2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cp /data/file1 /data/file2"],
pvolumes={"/data": step1.pvolume})
step3 = dsl.ContainerOp(
name="step3",
image="library/bash:4.4.23",
command=["cat", "/mnt/file1", "/mnt/file2"],
pvolumes={"/mnt": step2.pvolume})
def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]):
op0 = dsl.ContainerOp(
name="my-out-cop0",
image='python:alpine3.6',
command=["sh", "-c"],
arguments=[
'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'
],
file_outputs={'out': '/tmp/out.json'},
)
with dsl.ParallelFor(loopidy_doop) as item:
op1 = dsl.ContainerOp(
name="my-in-cop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo no output global op1, item.a: %s" % item.a],
).after(op0)
op_out = dsl.ContainerOp(
name="my-out-cop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo no output global op2, outp: %s" % op0.output],
)
def volume_snapshotop_sequential(url):
vop = dsl.VolumeOp(
name="create_volume",
resource_name="vol1",
size="1Gi",
modes=dsl.VOLUME_MODE_RWM
)
step1 = dsl.ContainerOp(
name="step1_ingest",
image="google/cloud-sdk:279.0.0",
command=["sh", "-c"],
arguments=["mkdir /data/step1 && "
"gsutil cat %s | gzip -c >/data/step1/file1.gz" % url],
pvolumes={"/data": vop.volume}
)
step1_snap = dsl.VolumeSnapshotOp(
name="step1_snap",
resource_name="step1_snap",
volume=step1.pvolume
)
step2 = dsl.ContainerOp(
name="step2_gunzip",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["mkdir /data/step2 && "
"gunzip /data/step1/file1.gz -c >/data/step2/file1"],
pvolumes={"/data": step1.pvolume}
)
step2_snap = dsl.VolumeSnapshotOp(
name="step2_snap",
resource_name="step2_snap",
volume=step2.pvolume
)
step3 = dsl.ContainerOp(
name="step3_copy",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["mkdir /data/step3 && "
"cp -av /data/step2/file1 /data/step3/file3"],
pvolumes={"/data": step2.pvolume}
)
step3_snap = dsl.VolumeSnapshotOp(
name="step3_snap",
resource_name="step3_snap",
volume=step3.pvolume
)
step4 = dsl.ContainerOp(
name="step4_output",
image="library/bash:4.4.23",
command=["cat", "/data/step2/file1", "/data/step3/file3"],
pvolumes={"/data": step3.pvolume}
)
def echo2_op(text1, text2):
return dsl.ContainerOp(
name='echo',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "Text 1: $0"; echo "Text 2: $1"', text1, text2]
)
def print_op(msg):
"""Print a message."""
return dsl.ContainerOp(
name='Print',
image='alpine:3.6',
command=['echo', msg],
)
def param_substitutions():
vop = dsl.VolumeOp(name="create_volume", resource_name="data", size="1Gi")
op = dsl.ContainerOp(name="cop",
image="image",
arguments=["--param", vop.output],
pvolumes={"/mnt": vop.volume})
def echo_op():
return dsl.ContainerOp(
name='echo',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "hello world"']
)
def param_substitutions(param):
dsl.get_pipeline_conf().op_transformers.append(add_common_labels(param))
op = dsl.ContainerOp(
name="cop",
image="image",
)
def echo_op(text):
return dsl.ContainerOp(
name='echo',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "$0"', text]
)
def sidecar_pipeline():
echo = dsl.Sidecar(name='echo',
image='hashicorp/http-echo',
args=['-text="hello world"'])
op1 = dsl.ContainerOp(
name='download',
image='busybox',
command=['sh', '-c'],
arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'],
sidecars=[echo],
file_outputs={'downloaded': '/tmp/results.txt'})
op2 = dsl.ContainerOp(name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo %s' % op1.output])
def gcs_download_op(url):
return dsl.ContainerOp(
name='GCS - Download',
image='google/cloud-sdk:279.0.0',
command=['sh', '-c'],
arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'],
file_outputs={
'data': '/tmp/results.txt',
}
)
def flip_coin_op():
"""Flip a coin and output heads or tails randomly."""
return dsl.ContainerOp(
name='Flip coin',
image='python:alpine3.6',
command=['sh', '-c'],
arguments=['python -c "import random; result = \'heads\' if random.randint(0,1) == 0 '
'else \'tails\'; print(result)" | tee /tmp/output'],
file_outputs={'output': '/tmp/output'}
)
def component_with_inline_input_artifact(text: str):
return dsl.ContainerOp(
name='component_with_inline_input_artifact',
image='alpine',
command=[
'cat',
dsl.InputArgumentPath(text,
path='/tmp/inputs/text/data',
input='text')
], # path and input are optional
)
def volume_snapshotop_rokurl(rok_url):
vop1 = dsl.VolumeOp(name="create_volume_1",
resource_name="vol1",
size="1Gi",
annotations={"rok/origin": rok_url},
modes=dsl.VOLUME_MODE_RWM)
step1 = dsl.ContainerOp(
name="step1_concat",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cat /data/file*| gzip -c >/data/full.gz"],
pvolumes={"/data": vop1.volume})
step1_snap = dsl.VolumeSnapshotOp(name="create_snapshot_1",
resource_name="snap1",
volume=step1.pvolume)
vop2 = dsl.VolumeOp(name="create_volume_2",
resource_name="vol2",
data_source=step1_snap.snapshot,
size=step1_snap.outputs["size"])
step2 = dsl.ContainerOp(name="step2_gunzip",
image="library/bash:4.4.23",
command=["gunzip", "-k", "/data/full.gz"],
pvolumes={"/data": vop2.volume})
step2_snap = dsl.VolumeSnapshotOp(name="create_snapshot_2",
resource_name="snap2",
volume=step2.pvolume)
vop3 = dsl.VolumeOp(name="create_volume_3",
resource_name="vol3",
data_source=step2_snap.snapshot,
size=step2_snap.outputs["size"])
step3 = dsl.ContainerOp(name="step3_output",
image="library/bash:4.4.23",
command=["cat", "/data/full"],
pvolumes={"/data": vop3.volume})
def volume_pipeline():
op1 = dsl.ContainerOp(
name='download',
image='google/cloud-sdk',
command=['sh', '-c'],
arguments=['ls | tee /tmp/results.txt'],
file_outputs={'downloaded': '/tmp/results.txt'}) \
.add_volume(k8s_client.V1Volume(name='gcp-credentials',
secret=k8s_client.V1SecretVolumeSource(
secret_name='user-gcp-sa'))) \
.add_volume_mount(k8s_client.V1VolumeMount(
mount_path='/secret/gcp-credentials', name='gcp-credentials')) \
.add_env_variable(k8s_client.V1EnvVar(
name='GOOGLE_APPLICATION_CREDENTIALS',
value='/secret/gcp-credentials/user-gcp-sa.json')) \
.add_env_variable(k8s_client.V1EnvVar(name='Foo', value='bar'))
op2 = dsl.ContainerOp(
name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo %s' % op1.output])
def component_with_input_artifact(text):
"""A component that passes text as input artifact."""
return dsl.ContainerOp(
name='component_with_input_artifact',
artifact_argument_paths=[
dsl.InputArgumentPath(argument=text,
path='/tmp/inputs/text/data',
input='text'), # path and input are optional
],
image='alpine',
command=['cat', '/tmp/inputs/text/data'],
)
def volumeop_basic(size):
vop = dsl.VolumeOp(
name="create_pvc",
resource_name="my-pvc",
modes=dsl.VOLUME_MODE_RWM,
size=size)
cop = dsl.ContainerOp(
name="cop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo foo > /mnt/file1"],
pvolumes={"/mnt": vop.volume})
def volume_op_dag():
vop = dsl.VolumeOp(name="create_pvc",
resource_name="my-pvc",
size="10Gi",
modes=dsl.VOLUME_MODE_RWM)
step1 = dsl.ContainerOp(name="step1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 1 | tee /mnt/file1"],
pvolumes={"/mnt": vop.volume})
step2 = dsl.ContainerOp(name="step2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 2 | tee /mnt2/file2"],
pvolumes={"/mnt2": vop.volume})
step3 = dsl.ContainerOp(name="step3",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cat /mnt/file1 /mnt/file2"],
pvolumes={"/mnt": vop.volume.after(step1, step2)})
def pipeline_with_sidecar():
# sidecar with sevice that reply "hello world" to any GET request
echo = dsl.Sidecar(
name="echo",
image="nginx:1.13",
command=["nginx", "-g", "daemon off;"],
)
# container op with sidecar
op1 = dsl.ContainerOp(
name="download",
image="busybox:latest",
command=["sh", "-c"],
arguments=[
"until wget http://localhost:80 -O /tmp/results.txt; do sleep 5; done && cat /tmp/results.txt"
],
sidecars=[echo],
file_outputs={"downloaded": "/tmp/results.txt"},
)
def resourceop_basic(username, password):
secret_resource = k8s_client.V1Secret(
api_version="v1",
kind="Secret",
metadata=k8s_client.V1ObjectMeta(generate_name="my-secret-"),
type="Opaque",
data={
"username": username,
"password": password
})
rop = dsl.ResourceOp(name="create-my-secret",
k8s_resource=secret_resource,
attribute_outputs={"name": "{.metadata.name}"})
secret = k8s_client.V1Volume(
name="my-secret",
secret=k8s_client.V1SecretVolumeSource(secret_name=rop.output))
cop = dsl.ContainerOp(name="cop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["ls /etc/secret-volume"],
pvolumes={"/etc/secret-volume": secret})
def echo2_graph_component(text2):
dsl.ContainerOp(name='echo2-task1',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "$0"', text2])
def my_pipeline():
v1dsl.ContainerOp(name='comp1',
image='gcr.io/dummy',
command=['python', 'main.py'])
