def volume_snapshotop_sequential(url):
vop = dsl.VolumeOp(
name="create_volume",
resource_name="vol1",
size="1Gi",
modes=dsl.VOLUME_MODE_RWM
)
step1 = dsl.ContainerOp(
name="step1_ingest",
image="google/cloud-sdk:279.0.0",
command=["sh", "-c"],
arguments=["mkdir /data/step1 && "
"gsutil cat %s | gzip -c >/data/step1/file1.gz" % url],
pvolumes={"/data": vop.volume}
)
step1_snap = dsl.VolumeSnapshotOp(
name="step1_snap",
resource_name="step1_snap",
volume=step1.pvolume
)
step2 = dsl.ContainerOp(
name="step2_gunzip",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["mkdir /data/step2 && "
"gunzip /data/step1/file1.gz -c >/data/step2/file1"],
pvolumes={"/data": step1.pvolume}
)
step2_snap = dsl.VolumeSnapshotOp(
name="step2_snap",
resource_name="step2_snap",
volume=step2.pvolume
)
step3 = dsl.ContainerOp(
name="step3_copy",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["mkdir /data/step3 && "
"cp -av /data/step2/file1 /data/step3/file3"],
pvolumes={"/data": step2.pvolume}
)
step3_snap = dsl.VolumeSnapshotOp(
name="step3_snap",
resource_name="step3_snap",
volume=step3.pvolume
)
step4 = dsl.ContainerOp(
name="step4_output",
image="library/bash:4.4.23",
command=["cat", "/data/step2/file1", "/data/step3/file3"],
pvolumes={"/data": step3.pvolume}
)
def volume_snapshotop_rokurl(rok_url):
vop1 = dsl.VolumeOp(
name="create_volume_1",
resource_name="vol1",
size="1Gi",
annotations={"rok/origin": rok_url},
modes=dsl.VOLUME_MODE_RWM)
step1 = dsl.ContainerOp(
name="step1_concat",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cat /data/file*| gzip -c >/data/full.gz"],
pvolumes={"/data": vop1.volume})
step1_snap = dsl.VolumeSnapshotOp(
name="create_snapshot_1", resource_name="snap1", volume=step1.pvolume)
vop2 = dsl.VolumeOp(
name="create_volume_2",
resource_name="vol2",
data_source=step1_snap.snapshot,
size=step1_snap.outputs["size"])
step2 = dsl.ContainerOp(
name="step2_gunzip",
image="library/bash:4.4.23",
command=["gunzip", "-k", "/data/full.gz"],
pvolumes={"/data": vop2.volume})
step2_snap = dsl.VolumeSnapshotOp(
name="create_snapshot_2", resource_name="snap2", volume=step2.pvolume)
vop3 = dsl.VolumeOp(
name="create_volume_3",
resource_name="vol3",
data_source=step2_snap.snapshot,
size=step2_snap.outputs["size"])
step3 = dsl.ContainerOp(
name="step3_output",
image="library/bash:4.4.23",
command=["cat", "/data/full"],
pvolumes={"/data": vop3.volume})