def test_omitting_name(self):
"""Test PipelineVolume creation when omitting "name"."""
vol1 = PipelineVolume(pvc="foo")
vol2 = PipelineVolume(name="provided", pvc="foo")
name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a114486e3"
"1216e083a563")
name2 = "provided"
self.assertEqual(vol1.name, name1)
self.assertEqual(vol2.name, name2)
def test_omitting_name(self):
"""Test PipelineVolume creation when omitting "name"."""
vol1 = PipelineVolume(pvc="foo")
vol2 = PipelineVolume(name="provided", pvc="foo")
name1 = ("pvolume-109601a31098e6c92e2ad294fee3fae234e3f52398d3b077b0e0"
"690e4cece686")
name2 = "provided"
self.assertEqual(vol1.name, name1)
self.assertEqual(vol2.name, name2)
def my_pipeline(param='foo'):
vol1 = PipelineVolume(pvc="foo")
vol2 = PipelineVolume(name="provided", pvc="foo")
name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a1144"
"86e3121")
name2 = "provided"
self.assertEqual(vol1.name, name1)
self.assertEqual(vol2.name, name2)
# Testing json.dumps() when pvc is a PipelineParam to avoid
# `TypeError: Object of type PipelineParam is not JSON serializable`
vol3 = PipelineVolume(pvc=param)
def my_pipeline(param='foo'):
vol1 = PipelineVolume(pvc="foo")
vol2 = PipelineVolume(name="provided", pvc="foo")
name1 = (
"pvolume-4cf668b8c7be134cfcbd7758d1eef9643d1bd7ed9925a98e707635b"
)
name2 = "provided"
self.assertEqual(vol1.name, name1)
self.assertEqual(vol2.name, name2)
# Testing json.dumps() when pvc is a PipelineParam to avoid
# `TypeError: Object of type PipelineParam is not JSON serializable`
vol3 = PipelineVolume(pvc=param)
def volume_op_dag():
datasetName = "your-dataset"
dataset = PipelineVolume(datasetName)
step0 = dsl.ResourceOp(name="dataset-creation",
k8s_resource=get_dataset_yaml(
datasetName, "XXXXXXXXXXXXXXX",
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"http://your_endpoint.com", "bucket-name", ""))
step1 = dsl.ContainerOp(name="step1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 1|tee /data/file1"],
pvolumes={
"/data": dataset
}).after(step0)
step2 = dsl.ContainerOp(name="step2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cp /data/file1 /data/file2"],
pvolumes={"/data": step1.pvolume})
step3 = dsl.ContainerOp(name="step3",
image="library/bash:4.4.23",
command=["cat", "/mnt/file1", "/mnt/file2"],
pvolumes={"/mnt": step2.pvolume})
def test_add_pvolumes(self):
pvolume = PipelineVolume(pvc='test')
op = ContainerOp(name='op1', image='image', pvolumes={'/mnt': pvolume})
self.assertEqual(pvolume.dependent_names, [])
self.assertEqual(op.pvolume.dependent_names, [op.name])
self.assertEqual(op.volumes[0].dependent_names, [op.name])
def my_pipeline(param1, param2):
vol = VolumeOp(name="myvol_creation",
resource_name=param1,
size=param2,
annotations={"test": "annotation"})
self.assertCountEqual([x.name for x in vol.inputs],
["param1", "param2"])
self.assertEqual(vol.k8s_resource.metadata.name,
"{{workflow.name}}-%s" % str(param1))
expected_attribute_outputs = {
"manifest": "{}",
"name": "{.metadata.name}",
"size": "{.status.capacity.storage}"
}
self.assertEqual(vol.attribute_outputs, expected_attribute_outputs)
expected_outputs = {
"manifest": PipelineParam(name="manifest", op_name=vol.name),
"name": PipelineParam(name="name", op_name=vol.name),
"size": PipelineParam(name="size", op_name=vol.name)
}
self.assertEqual(vol.outputs, expected_outputs)
self.assertEqual(vol.output,
PipelineParam(name="name", op_name=vol.name))
self.assertEqual(vol.dependent_names, [])
expected_volume = PipelineVolume(
name="myvol-creation",
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
claim_name=PipelineParam(name="name", op_name=vol.name)))
def my_pipeline():
op1 = ContainerOp(name="op1", image="image")
op2 = ContainerOp(name="op2", image="image").after(op1)
op3 = ContainerOp(name="op3", image="image")
vol1 = PipelineVolume(name="pipeline-volume")
vol2 = vol1.after(op1)
vol3 = vol2.after(op2)
vol4 = vol3.after(op1, op2)
vol5 = vol4.after(op3)
self.assertEqual(vol1.dependent_names, [])
self.assertEqual(vol2.dependent_names, [op1.name])
self.assertEqual(vol3.dependent_names, [op2.name])
self.assertEqual(sorted(vol4.dependent_names),
[op1.name, op2.name])
self.assertEqual(sorted(vol5.dependent_names),
[op1.name, op2.name, op3.name])
def test_after_method(self):
"""Test the after method."""
with Pipeline("somename") as p:
op1 = ContainerOp(name="op1", image="image")
op2 = ContainerOp(name="op2", image="image").after(op1)
op3 = ContainerOp(name="op3", image="image")
vol1 = PipelineVolume(name="pipeline-volume")
vol2 = vol1.after(op1)
vol3 = vol2.after(op2)
vol4 = vol3.after(op1, op2)
vol5 = vol4.after(op3)
self.assertEqual(vol1.dependent_names, [])
self.assertEqual(vol2.dependent_names, [op1.name])
self.assertEqual(vol3.dependent_names, [op2.name])
self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name])
self.assertEqual(sorted(vol5.dependent_names),
[op1.name, op2.name, op3.name])
def volume_op_dag():
dataset = PipelineVolume("your-dataset")
step1 = dsl.ContainerOp(name="step1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo 1|tee /data/file1"],
pvolumes={"/data": dataset})
step2 = dsl.ContainerOp(name="step2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["cp /data/file1 /data/file2"],
pvolumes={"/data": step1.pvolume})
step3 = dsl.ContainerOp(name="step3",
image="library/bash:4.4.23",
command=["cat", "/mnt/file1", "/mnt/file2"],
pvolumes={"/mnt": step2.pvolume})