def test_omitting_name(self): """Test PipelineVolume creation when omitting "name".""" vol1 = PipelineVolume(pvc="foo") vol2 = PipelineVolume(name="provided", pvc="foo") name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a114486e3" "1216e083a563") name2 = "provided" self.assertEqual(vol1.name, name1) self.assertEqual(vol2.name, name2)
def test_omitting_name(self): """Test PipelineVolume creation when omitting "name".""" vol1 = PipelineVolume(pvc="foo") vol2 = PipelineVolume(name="provided", pvc="foo") name1 = ("pvolume-109601a31098e6c92e2ad294fee3fae234e3f52398d3b077b0e0" "690e4cece686") name2 = "provided" self.assertEqual(vol1.name, name1) self.assertEqual(vol2.name, name2)
def my_pipeline(param='foo'): vol1 = PipelineVolume(pvc="foo") vol2 = PipelineVolume(name="provided", pvc="foo") name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a1144" "86e3121") name2 = "provided" self.assertEqual(vol1.name, name1) self.assertEqual(vol2.name, name2) # Testing json.dumps() when pvc is a PipelineParam to avoid # `TypeError: Object of type PipelineParam is not JSON serializable` vol3 = PipelineVolume(pvc=param)
def my_pipeline(param='foo'): vol1 = PipelineVolume(pvc="foo") vol2 = PipelineVolume(name="provided", pvc="foo") name1 = ( "pvolume-4cf668b8c7be134cfcbd7758d1eef9643d1bd7ed9925a98e707635b" ) name2 = "provided" self.assertEqual(vol1.name, name1) self.assertEqual(vol2.name, name2) # Testing json.dumps() when pvc is a PipelineParam to avoid # `TypeError: Object of type PipelineParam is not JSON serializable` vol3 = PipelineVolume(pvc=param)
def volume_op_dag(): datasetName = "your-dataset" dataset = PipelineVolume(datasetName) step0 = dsl.ResourceOp(name="dataset-creation", k8s_resource=get_dataset_yaml( datasetName, "XXXXXXXXXXXXXXX", "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "http://your_endpoint.com", "bucket-name", "")) step1 = dsl.ContainerOp(name="step1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 1|tee /data/file1"], pvolumes={ "/data": dataset }).after(step0) step2 = dsl.ContainerOp(name="step2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["cp /data/file1 /data/file2"], pvolumes={"/data": step1.pvolume}) step3 = dsl.ContainerOp(name="step3", image="library/bash:4.4.23", command=["cat", "/mnt/file1", "/mnt/file2"], pvolumes={"/mnt": step2.pvolume})
def test_add_pvolumes(self): pvolume = PipelineVolume(pvc='test') op = ContainerOp(name='op1', image='image', pvolumes={'/mnt': pvolume}) self.assertEqual(pvolume.dependent_names, []) self.assertEqual(op.pvolume.dependent_names, [op.name]) self.assertEqual(op.volumes[0].dependent_names, [op.name])
def my_pipeline(param1, param2): vol = VolumeOp(name="myvol_creation", resource_name=param1, size=param2, annotations={"test": "annotation"}) self.assertCountEqual([x.name for x in vol.inputs], ["param1", "param2"]) self.assertEqual(vol.k8s_resource.metadata.name, "{{workflow.name}}-%s" % str(param1)) expected_attribute_outputs = { "manifest": "{}", "name": "{.metadata.name}", "size": "{.status.capacity.storage}" } self.assertEqual(vol.attribute_outputs, expected_attribute_outputs) expected_outputs = { "manifest": PipelineParam(name="manifest", op_name=vol.name), "name": PipelineParam(name="name", op_name=vol.name), "size": PipelineParam(name="size", op_name=vol.name) } self.assertEqual(vol.outputs, expected_outputs) self.assertEqual(vol.output, PipelineParam(name="name", op_name=vol.name)) self.assertEqual(vol.dependent_names, []) expected_volume = PipelineVolume( name="myvol-creation", persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name=PipelineParam(name="name", op_name=vol.name)))
def my_pipeline(): op1 = ContainerOp(name="op1", image="image") op2 = ContainerOp(name="op2", image="image").after(op1) op3 = ContainerOp(name="op3", image="image") vol1 = PipelineVolume(name="pipeline-volume") vol2 = vol1.after(op1) vol3 = vol2.after(op2) vol4 = vol3.after(op1, op2) vol5 = vol4.after(op3) self.assertEqual(vol1.dependent_names, []) self.assertEqual(vol2.dependent_names, [op1.name]) self.assertEqual(vol3.dependent_names, [op2.name]) self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name]) self.assertEqual(sorted(vol5.dependent_names), [op1.name, op2.name, op3.name])
def test_after_method(self): """Test the after method.""" with Pipeline("somename") as p: op1 = ContainerOp(name="op1", image="image") op2 = ContainerOp(name="op2", image="image").after(op1) op3 = ContainerOp(name="op3", image="image") vol1 = PipelineVolume(name="pipeline-volume") vol2 = vol1.after(op1) vol3 = vol2.after(op2) vol4 = vol3.after(op1, op2) vol5 = vol4.after(op3) self.assertEqual(vol1.dependent_names, []) self.assertEqual(vol2.dependent_names, [op1.name]) self.assertEqual(vol3.dependent_names, [op2.name]) self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name]) self.assertEqual(sorted(vol5.dependent_names), [op1.name, op2.name, op3.name])
def volume_op_dag(): dataset = PipelineVolume("your-dataset") step1 = dsl.ContainerOp(name="step1", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["echo 1|tee /data/file1"], pvolumes={"/data": dataset}) step2 = dsl.ContainerOp(name="step2", image="library/bash:4.4.23", command=["sh", "-c"], arguments=["cp /data/file1 /data/file2"], pvolumes={"/data": step1.pvolume}) step3 = dsl.ContainerOp(name="step3", image="library/bash:4.4.23", command=["cat", "/mnt/file1", "/mnt/file2"], pvolumes={"/mnt": step2.pvolume})