Ejemplo n.º 1
0
 def test_omitting_name(self):
     """Test PipelineVolume creation when omitting "name"."""
     vol1 = PipelineVolume(pvc="foo")
     vol2 = PipelineVolume(name="provided", pvc="foo")
     name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a114486e3"
              "1216e083a563")
     name2 = "provided"
     self.assertEqual(vol1.name, name1)
     self.assertEqual(vol2.name, name2)
Ejemplo n.º 2
0
    def test_omitting_name(self):
        """Test PipelineVolume creation when omitting "name"."""
        vol1 = PipelineVolume(pvc="foo")
        vol2 = PipelineVolume(name="provided", pvc="foo")
        name1 = ("pvolume-109601a31098e6c92e2ad294fee3fae234e3f52398d3b077b0e0"
                 "690e4cece686")
        name2 = "provided"

        self.assertEqual(vol1.name, name1)
        self.assertEqual(vol2.name, name2)
Ejemplo n.º 3
0
        def my_pipeline(param='foo'):
            vol1 = PipelineVolume(pvc="foo")
            vol2 = PipelineVolume(name="provided", pvc="foo")
            name1 = ("pvolume-127ac63cf2013e9b95c192eb6a2c7d5a023ebeb51f6a1144"
                     "86e3121")
            name2 = "provided"
            self.assertEqual(vol1.name, name1)
            self.assertEqual(vol2.name, name2)

            # Testing json.dumps() when pvc is a PipelineParam to avoid
            # `TypeError: Object of type PipelineParam is not JSON serializable`
            vol3 = PipelineVolume(pvc=param)
Ejemplo n.º 4
0
        def my_pipeline(param='foo'):
            vol1 = PipelineVolume(pvc="foo")
            vol2 = PipelineVolume(name="provided", pvc="foo")
            name1 = (
                "pvolume-4cf668b8c7be134cfcbd7758d1eef9643d1bd7ed9925a98e707635b"
            )
            name2 = "provided"
            self.assertEqual(vol1.name, name1)
            self.assertEqual(vol2.name, name2)

            # Testing json.dumps() when pvc is a PipelineParam to avoid
            # `TypeError: Object of type PipelineParam is not JSON serializable`
            vol3 = PipelineVolume(pvc=param)
Ejemplo n.º 5
0
def volume_op_dag():

    datasetName = "your-dataset"
    dataset = PipelineVolume(datasetName)

    step0 = dsl.ResourceOp(name="dataset-creation",
                           k8s_resource=get_dataset_yaml(
                               datasetName, "XXXXXXXXXXXXXXX",
                               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
                               "http://your_endpoint.com", "bucket-name", ""))

    step1 = dsl.ContainerOp(name="step1",
                            image="library/bash:4.4.23",
                            command=["sh", "-c"],
                            arguments=["echo 1|tee /data/file1"],
                            pvolumes={
                                "/data": dataset
                            }).after(step0)

    step2 = dsl.ContainerOp(name="step2",
                            image="library/bash:4.4.23",
                            command=["sh", "-c"],
                            arguments=["cp /data/file1 /data/file2"],
                            pvolumes={"/data": step1.pvolume})

    step3 = dsl.ContainerOp(name="step3",
                            image="library/bash:4.4.23",
                            command=["cat", "/mnt/file1", "/mnt/file2"],
                            pvolumes={"/mnt": step2.pvolume})
Ejemplo n.º 6
0
    def test_add_pvolumes(self):
        pvolume = PipelineVolume(pvc='test')
        op = ContainerOp(name='op1', image='image', pvolumes={'/mnt': pvolume})

        self.assertEqual(pvolume.dependent_names, [])
        self.assertEqual(op.pvolume.dependent_names, [op.name])
        self.assertEqual(op.volumes[0].dependent_names, [op.name])
Ejemplo n.º 7
0
        def my_pipeline(param1, param2):
            vol = VolumeOp(name="myvol_creation",
                           resource_name=param1,
                           size=param2,
                           annotations={"test": "annotation"})

            self.assertCountEqual([x.name for x in vol.inputs],
                                  ["param1", "param2"])
            self.assertEqual(vol.k8s_resource.metadata.name,
                             "{{workflow.name}}-%s" % str(param1))
            expected_attribute_outputs = {
                "manifest": "{}",
                "name": "{.metadata.name}",
                "size": "{.status.capacity.storage}"
            }
            self.assertEqual(vol.attribute_outputs, expected_attribute_outputs)
            expected_outputs = {
                "manifest": PipelineParam(name="manifest", op_name=vol.name),
                "name": PipelineParam(name="name", op_name=vol.name),
                "size": PipelineParam(name="size", op_name=vol.name)
            }
            self.assertEqual(vol.outputs, expected_outputs)
            self.assertEqual(vol.output,
                             PipelineParam(name="name", op_name=vol.name))
            self.assertEqual(vol.dependent_names, [])
            expected_volume = PipelineVolume(
                name="myvol-creation",
                persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
                    claim_name=PipelineParam(name="name", op_name=vol.name)))
Ejemplo n.º 8
0
        def my_pipeline():
            op1 = ContainerOp(name="op1", image="image")
            op2 = ContainerOp(name="op2", image="image").after(op1)
            op3 = ContainerOp(name="op3", image="image")
            vol1 = PipelineVolume(name="pipeline-volume")
            vol2 = vol1.after(op1)
            vol3 = vol2.after(op2)
            vol4 = vol3.after(op1, op2)
            vol5 = vol4.after(op3)

            self.assertEqual(vol1.dependent_names, [])
            self.assertEqual(vol2.dependent_names, [op1.name])
            self.assertEqual(vol3.dependent_names, [op2.name])
            self.assertEqual(sorted(vol4.dependent_names),
                             [op1.name, op2.name])
            self.assertEqual(sorted(vol5.dependent_names),
                             [op1.name, op2.name, op3.name])
Ejemplo n.º 9
0
    def test_after_method(self):
        """Test the after method."""
        with Pipeline("somename") as p:
            op1 = ContainerOp(name="op1", image="image")
            op2 = ContainerOp(name="op2", image="image").after(op1)
            op3 = ContainerOp(name="op3", image="image")
            vol1 = PipelineVolume(name="pipeline-volume")
            vol2 = vol1.after(op1)
            vol3 = vol2.after(op2)
            vol4 = vol3.after(op1, op2)
            vol5 = vol4.after(op3)

        self.assertEqual(vol1.dependent_names, [])
        self.assertEqual(vol2.dependent_names, [op1.name])
        self.assertEqual(vol3.dependent_names, [op2.name])
        self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name])
        self.assertEqual(sorted(vol5.dependent_names),
                         [op1.name, op2.name, op3.name])
Ejemplo n.º 10
0
def volume_op_dag():

    dataset = PipelineVolume("your-dataset")

    step1 = dsl.ContainerOp(name="step1",
                            image="library/bash:4.4.23",
                            command=["sh", "-c"],
                            arguments=["echo 1|tee /data/file1"],
                            pvolumes={"/data": dataset})

    step2 = dsl.ContainerOp(name="step2",
                            image="library/bash:4.4.23",
                            command=["sh", "-c"],
                            arguments=["cp /data/file1 /data/file2"],
                            pvolumes={"/data": step1.pvolume})

    step3 = dsl.ContainerOp(name="step3",
                            image="library/bash:4.4.23",
                            command=["cat", "/mnt/file1", "/mnt/file2"],
                            pvolumes={"/mnt": step2.pvolume})