def test_handle_parsing_predicates(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
task 2:
componentRef: {name: Comp 2}
arguments:
in2 1: 21
in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}}
isEnabled:
not:
and:
op1:
'>':
op1: {taskOutput: {taskId: task 1, outputName: out1 1}}
op2: 0
op2:
'==':
op1: {taskOutput: {taskId: task 1, outputName: out1 2}}
op2: 'head'
'''
struct = load_yaml(component_text)
ComponentSpec.from_dict(struct)
def test_handle_parsing_graph_component(self):
component_text = '''\
inputs:
- {name: graph in 1}
- {name: graph in 2}
outputs:
- {name: graph out 1}
- {name: graph out 2}
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
arguments:
in1 1: 11
task 2:
componentRef: {name: Comp 2}
arguments:
in2 1: 21
in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}}
task 3:
componentRef: {name: Comp 3}
arguments:
in3 1: {taskOutput: {taskId: task 2, outputName: out2 1}}
in3 2: {graphInput: {inputName: graph in 1}}
outputValues:
graph out 1: {taskOutput: {taskId: task 3, outputName: out3 1}}
graph out 2: {taskOutput: {taskId: task 1, outputName: out1 2}}
'''
struct = load_yaml(component_text)
ComponentSpec.from_dict(struct)
def test_handle_parsing_task_volumes_and_mounts(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
executionOptions:
kubernetesOptions:
mainContainer:
volumeMounts:
- name: workdir
mountPath: /mnt/vol
podSpec:
volumes:
- name: workdir
emptyDir: {}
'''
struct = load_yaml(component_text)
component_spec = ComponentSpec.from_dict(struct)
self.assertEqual(
component_spec.implementation.graph.tasks['task 1'].
execution_options.kubernetes_options.pod_spec.volumes[0].name,
'workdir')
self.assertIsNotNone(
component_spec.implementation.graph.tasks['task 1'].
execution_options.kubernetes_options.pod_spec.volumes[0].empty_dir)
def test_loading_minimal_component(self):
component_text = '''\
implementation:
container:
image: busybox
'''
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component(text=component_text)
task1 = task_factory1()
assert task1.image == component_dict['implementation']['container']['image']
def test_loading_minimal_component(self):
component_text = '''\
implementation:
container:
image: busybox
'''
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component(text=component_text)
self.assertEqual(
task_factory1.component_spec.implementation.container.image,
component_dict['implementation']['container']['image'])
def _load_component_spec_from_component_text(text) -> structures.ComponentSpec:
component_dict = load_yaml(text)
component_spec = structures.ComponentSpec.from_dict(component_dict)
# Calculating hash digest for the component
import hashlib
data = text if isinstance(text, bytes) else text.encode('utf-8')
data = data.replace(b'\r\n', b'\n') # Normalizing line endings
digest = hashlib.sha256(data).hexdigest()
component_spec._digest = digest
return component_spec
def test_accessing_component_spec_from_task_factory(self):
component_text = '''\
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
actual_component_spec = task_factory1.component_spec
actual_component_spec_dict = actual_component_spec.to_dict()
expected_component_spec_dict = load_yaml(component_text)
expected_component_spec = kfp.components._structures.ComponentSpec.from_dict(expected_component_spec_dict)
self.assertEqual(expected_component_spec_dict, actual_component_spec_dict)
self.assertEqual(expected_component_spec, task_factory1.component_spec)
def dkube_op(name, token, stage, **kwargs):
assert stage in VALID_STAGES, "Invalid value for stage, must be one of training/preprocessing/serving/custom"
component = None
path = Path(__file__).parent
with open('{}/dkube.yaml'.format(path), 'rb') as stream:
cdict = load_yaml(stream)
cdict['name'] = name
cidct['metadata']['labels']['stage'] = stage
cyaml = dump_yaml(cdict)
component = components.load_component_from_text(cyaml)
assert component != None, "Internal error, loading DKube component failed"
return component(name, token, stage, **kwargs)
def test_fail_on_cyclic_references(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
arguments:
in1 1: {taskOutput: {taskId: task 2, outputName: out2 1}}
task 2:
componentRef: {name: Comp 2}
arguments:
in2 1: {taskOutput: {taskId: task 1, outputName: out1 1}}
'''
struct = load_yaml(component_text)
ComponentSpec.from_dict(struct)
def test_handle_parsing_task_execution_options_caching_strategy(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
executionOptions:
cachingStrategy:
maxCacheStaleness: P30D
'''
struct = load_yaml(component_text)
component_spec = ComponentSpec.from_dict(struct)
self.assertEqual(
component_spec.implementation.graph.tasks['task 1'].
execution_options.caching_strategy.max_cache_staleness, 'P30D')
def test_handle_parsing_task_container_spec_options(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
k8sContainerOptions:
resources:
requests:
memory: 1024Mi
cpu: 200m
'''
struct = load_yaml(component_text)
component_spec = ComponentSpec.from_dict(struct)
self.assertEqual(component_spec.implementation.graph.tasks['task 1'].k8s_container_options.resources.requests['memory'], '1024Mi')
def test_load_component_from_file(self):
component_path_obj = _test_data_dir.joinpath(
'python_add.component.yaml')
component_text = component_path_obj.read_text()
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_file(str(component_path_obj))
assert task_factory1.__doc__ == component_dict['description']
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
assert task1.human_name == component_dict['name']
assert task1.image == component_dict['implementation'][
'dockerContainer']['image']
assert task1.arguments[0] == str(arg1)
assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self):
url = 'https://raw.githubusercontent.com/kubeflow/pipelines/638045974d688b473cda9f4516a2cf1d7d1e02dd/sdk/python/tests/components/test_data/python_add.component.yaml'
import requests
resp = requests.get(url)
component_text = resp.content
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_url(url)
assert task_factory1.__doc__ == component_dict['description']
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
assert task1.human_name == component_dict['name']
assert task1.image == component_dict['implementation']['container']['image']
assert task1.arguments[0] == str(arg1)
assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self):
url = 'https://raw.githubusercontent.com/kubeflow/pipelines/eb830cd73ca148e5a1a6485a9374c2dc068314bc/sdk/python/tests/components/test_data/python_add.component.yaml'
import requests
resp = requests.get(url)
component_text = resp.content
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_url(url)
assert task_factory1.__doc__ == component_dict['name'] + '\n' + component_dict['description']
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
assert task1.human_name == component_dict['name']
assert task1.container.image == component_dict['implementation']['container']['image']
assert task1.arguments[0] == str(arg1)
assert task1.arguments[1] == str(arg2)
def test_load_component_from_url(self):
url = 'https://raw.githubusercontent.com/kubeflow/pipelines/e54fe675432cfef1d115a7a2909f08ed95ea8933/sdk/python/tests/components/test_data/python_add.component.yaml'
import requests
resp = requests.get(url)
component_text = resp.content
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_url(url)
self.assertEqual(task_factory1.__doc__, component_dict['name'] + '\n' + component_dict['description'])
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task1.human_name, component_dict['name'])
self.assertEqual(task1.container.image, component_dict['implementation']['container']['image'])
self.assertEqual(task1.arguments[0], str(arg1))
self.assertEqual(task1.arguments[1], str(arg2))
def test_handle_parsing_task_volumes_and_mounts(self):
component_text = '''\
implementation:
graph:
tasks:
task 1:
componentRef: {name: Comp 1}
k8sContainerOptions:
volumeMounts:
- name: workdir
mountPath: /mnt/vol
k8sPodOptions:
spec:
volumes:
- name: workdir
emptyDir: {}
'''
struct = load_yaml(component_text)
component_spec = ComponentSpec.from_dict(struct)
self.assertEqual(component_spec.implementation.graph.tasks['task 1'].k8s_pod_options.spec.volumes[0].name, 'workdir')
self.assertTrue(component_spec.implementation.graph.tasks['task 1'].k8s_pod_options.spec.volumes[0].empty_dir is not None)