def test_handle_parsing_predicates(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} task 2: componentRef: {name: Comp 2} arguments: in2 1: 21 in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}} isEnabled: not: and: op1: '>': op1: {taskOutput: {taskId: task 1, outputName: out1 1}} op2: 0 op2: '==': op1: {taskOutput: {taskId: task 1, outputName: out1 2}} op2: 'head' ''' struct = load_yaml(component_text) ComponentSpec.from_dict(struct)
def test_handle_parsing_graph_component(self): component_text = '''\ inputs: - {name: graph in 1} - {name: graph in 2} outputs: - {name: graph out 1} - {name: graph out 2} implementation: graph: tasks: task 1: componentRef: {name: Comp 1} arguments: in1 1: 11 task 2: componentRef: {name: Comp 2} arguments: in2 1: 21 in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}} task 3: componentRef: {name: Comp 3} arguments: in3 1: {taskOutput: {taskId: task 2, outputName: out2 1}} in3 2: {graphInput: {inputName: graph in 1}} outputValues: graph out 1: {taskOutput: {taskId: task 3, outputName: out3 1}} graph out 2: {taskOutput: {taskId: task 1, outputName: out1 2}} ''' struct = load_yaml(component_text) ComponentSpec.from_dict(struct)
def test_handle_parsing_task_volumes_and_mounts(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} executionOptions: kubernetesOptions: mainContainer: volumeMounts: - name: workdir mountPath: /mnt/vol podSpec: volumes: - name: workdir emptyDir: {} ''' struct = load_yaml(component_text) component_spec = ComponentSpec.from_dict(struct) self.assertEqual( component_spec.implementation.graph.tasks['task 1']. execution_options.kubernetes_options.pod_spec.volumes[0].name, 'workdir') self.assertIsNotNone( component_spec.implementation.graph.tasks['task 1']. execution_options.kubernetes_options.pod_spec.volumes[0].empty_dir)
def test_loading_minimal_component(self): component_text = '''\ implementation: container: image: busybox ''' component_dict = load_yaml(component_text) task_factory1 = comp.load_component(text=component_text) task1 = task_factory1() assert task1.image == component_dict['implementation']['container']['image']
def test_loading_minimal_component(self): component_text = '''\ implementation: container: image: busybox ''' component_dict = load_yaml(component_text) task_factory1 = comp.load_component(text=component_text) self.assertEqual( task_factory1.component_spec.implementation.container.image, component_dict['implementation']['container']['image'])
def _load_component_spec_from_component_text(text) -> structures.ComponentSpec: component_dict = load_yaml(text) component_spec = structures.ComponentSpec.from_dict(component_dict) # Calculating hash digest for the component import hashlib data = text if isinstance(text, bytes) else text.encode('utf-8') data = data.replace(b'\r\n', b'\n') # Normalizing line endings digest = hashlib.sha256(data).hexdigest() component_spec._digest = digest return component_spec
def test_accessing_component_spec_from_task_factory(self): component_text = '''\ implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text) actual_component_spec = task_factory1.component_spec actual_component_spec_dict = actual_component_spec.to_dict() expected_component_spec_dict = load_yaml(component_text) expected_component_spec = kfp.components._structures.ComponentSpec.from_dict(expected_component_spec_dict) self.assertEqual(expected_component_spec_dict, actual_component_spec_dict) self.assertEqual(expected_component_spec, task_factory1.component_spec)
def dkube_op(name, token, stage, **kwargs): assert stage in VALID_STAGES, "Invalid value for stage, must be one of training/preprocessing/serving/custom" component = None path = Path(__file__).parent with open('{}/dkube.yaml'.format(path), 'rb') as stream: cdict = load_yaml(stream) cdict['name'] = name cidct['metadata']['labels']['stage'] = stage cyaml = dump_yaml(cdict) component = components.load_component_from_text(cyaml) assert component != None, "Internal error, loading DKube component failed" return component(name, token, stage, **kwargs)
def test_fail_on_cyclic_references(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} arguments: in1 1: {taskOutput: {taskId: task 2, outputName: out2 1}} task 2: componentRef: {name: Comp 2} arguments: in2 1: {taskOutput: {taskId: task 1, outputName: out1 1}} ''' struct = load_yaml(component_text) ComponentSpec.from_dict(struct)
def test_handle_parsing_task_execution_options_caching_strategy(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} executionOptions: cachingStrategy: maxCacheStaleness: P30D ''' struct = load_yaml(component_text) component_spec = ComponentSpec.from_dict(struct) self.assertEqual( component_spec.implementation.graph.tasks['task 1']. execution_options.caching_strategy.max_cache_staleness, 'P30D')
def test_handle_parsing_task_container_spec_options(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} k8sContainerOptions: resources: requests: memory: 1024Mi cpu: 200m ''' struct = load_yaml(component_text) component_spec = ComponentSpec.from_dict(struct) self.assertEqual(component_spec.implementation.graph.tasks['task 1'].k8s_container_options.resources.requests['memory'], '1024Mi')
def test_load_component_from_file(self): component_path_obj = _test_data_dir.joinpath( 'python_add.component.yaml') component_text = component_path_obj.read_text() component_dict = load_yaml(component_text) task_factory1 = comp.load_component_from_file(str(component_path_obj)) assert task_factory1.__doc__ == component_dict['description'] arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) assert task1.human_name == component_dict['name'] assert task1.image == component_dict['implementation'][ 'dockerContainer']['image'] assert task1.arguments[0] == str(arg1) assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self): url = 'https://raw.githubusercontent.com/kubeflow/pipelines/638045974d688b473cda9f4516a2cf1d7d1e02dd/sdk/python/tests/components/test_data/python_add.component.yaml' import requests resp = requests.get(url) component_text = resp.content component_dict = load_yaml(component_text) task_factory1 = comp.load_component_from_url(url) assert task_factory1.__doc__ == component_dict['description'] arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) assert task1.human_name == component_dict['name'] assert task1.image == component_dict['implementation']['container']['image'] assert task1.arguments[0] == str(arg1) assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self): url = 'https://raw.githubusercontent.com/kubeflow/pipelines/eb830cd73ca148e5a1a6485a9374c2dc068314bc/sdk/python/tests/components/test_data/python_add.component.yaml' import requests resp = requests.get(url) component_text = resp.content component_dict = load_yaml(component_text) task_factory1 = comp.load_component_from_url(url) assert task_factory1.__doc__ == component_dict['name'] + '\n' + component_dict['description'] arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) assert task1.human_name == component_dict['name'] assert task1.container.image == component_dict['implementation']['container']['image'] assert task1.arguments[0] == str(arg1) assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self): url = 'https://raw.githubusercontent.com/kubeflow/pipelines/e54fe675432cfef1d115a7a2909f08ed95ea8933/sdk/python/tests/components/test_data/python_add.component.yaml' import requests resp = requests.get(url) component_text = resp.content component_dict = load_yaml(component_text) task_factory1 = comp.load_component_from_url(url) self.assertEqual(task_factory1.__doc__, component_dict['name'] + '\n' + component_dict['description']) arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) self.assertEqual(task1.human_name, component_dict['name']) self.assertEqual(task1.container.image, component_dict['implementation']['container']['image']) self.assertEqual(task1.arguments[0], str(arg1)) self.assertEqual(task1.arguments[1], str(arg2))
def test_handle_parsing_task_volumes_and_mounts(self): component_text = '''\ implementation: graph: tasks: task 1: componentRef: {name: Comp 1} k8sContainerOptions: volumeMounts: - name: workdir mountPath: /mnt/vol k8sPodOptions: spec: volumes: - name: workdir emptyDir: {} ''' struct = load_yaml(component_text) component_spec = ComponentSpec.from_dict(struct) self.assertEqual(component_spec.implementation.graph.tasks['task 1'].k8s_pod_options.spec.volumes[0].name, 'workdir') self.assertTrue(component_spec.implementation.graph.tasks['task 1'].k8s_pod_options.spec.volumes[0].empty_dir is not None)