def _assert_compiled_pipeline_equals_golden(
self, kfp_compiler: compiler.Compiler, pipeline_func: Callable,
golden_yaml_filename: str):
compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml')
kfp_compiler.compile(pipeline_func, package_path=compiled_file)
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_file = os.path.join(test_data_dir, golden_yaml_filename)
# Uncomment the following to update goldens.
# TODO: place this behind some --update_goldens flag.
# kfp_compiler.compile(pipeline_func, package_path=golden_file)
with open(golden_file, 'r') as f:
golden = yaml.safe_load(f)
with open(compiled_file, 'r') as f:
compiled = yaml.safe_load(f)
for workflow in golden, compiled:
del workflow['metadata']
for template in workflow['spec']['templates']:
template.pop('metadata', None)
self.maxDiff = None
self.assertDictEqual(golden, compiled)
def _assert_compiled_pipeline_equals_golden(self,
kfp_compiler: compiler.Compiler,
pipeline_func: Callable,
golden_yaml_filename: str):
compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml')
kfp_compiler.compile(pipeline_func, package_path=compiled_file)
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_file = os.path.join(test_data_dir, golden_yaml_filename)
# Uncomment the following to update goldens.
# TODO: place this behind some --update_goldens flag.
# kfp_compiler.compile(pipeline_func, package_path=golden_file)
with open(golden_file, 'r') as f:
golden = yaml.safe_load(f)
with open(compiled_file, 'r') as f:
compiled = yaml.safe_load(f)
for workflow in golden, compiled:
del workflow['metadata']
for template in workflow['spec']['templates']:
template.pop('metadata', None)
if 'initContainers' not in template:
continue
# Strip off the launcher image label before comparison
for initContainer in template['initContainers']:
initContainer['image'] = initContainer['image'].split(':')[0]
self.maxDiff = None
self.assertDictEqual(golden, compiled)
def _assert_compiled_pipeline_equals_golden(self,
kfp_compiler: compiler.Compiler,
pipeline_func: Callable,
golden_yaml_filename: str):
compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml')
kfp_compiler.compile(pipeline_func, package_path=compiled_file)
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
golden_file = os.path.join(test_data_dir, golden_yaml_filename)
def _load_compiled_template(filename: str) -> Dict:
with open(filename, 'r') as f:
workflow = yaml.safe_load(f)
del workflow['metadata']
for template in workflow['spec']['templates']:
template.pop('metadata', None)
if 'initContainers' not in template:
continue
# Strip off the launcher image label before comparison
for initContainer in template['initContainers']:
initContainer['image'] = initContainer['image'].split(
':')[0]
if 'container' in template:
template['container'] = json.loads(
re.sub("'kfp==(\d+).(\d+).(\d+)'", 'kfp',
json.dumps(template['container'])))
return workflow
golden = _load_compiled_template(golden_file)
compiled = _load_compiled_template(compiled_file)
# Devs can run the following command to update golden files:
# UPDATE_GOLDENS=True python3 -m unittest kfp/compiler/v2_compatible_compiler_test.py
# If UPDATE_GOLDENS=True, and the diff is
# different, update the golden file and reload it.
update_goldens = os.environ.get('UPDATE_GOLDENS', False)
if golden != compiled and update_goldens:
kfp_compiler.compile(pipeline_func, package_path=golden_file)
golden = _load_compiled_template(golden_file)
self.assertDictEqual(golden, compiled)
import kfp
from kfp import dsl
from kfp.compiler import Compiler
#sb_op = kfp.components.load_component_from_url("https://raw.githubusercontent.com/demotto/my-pipeline/master/expr001/component.yaml")
import requests
url = "https://raw.githubusercontent.com/demotto/my-pipeline/master/expr001/component.yaml"
resp = requests.get(url)
sb_op = kfp.components.load_component_from_text(resp.text)
@dsl.pipeline(
name='simple pipeline',
description='A trainer that does end-to-end distributed training for XGBoost models.'
)
def my_pipeline():
my_task = sb_op(
input1="xxxxxxxxxx",
input2="oooooooooo"
)
if __name__ == '__main__':
compiler = Compiler()
compiler.compile(my_pipeline, "my_brick.yaml")
import lady_gaga
def my_func():
lady_gaga.my_func_impl()
def next_func(a):
lady_gaga.next_func_impl()
my_op = components.func_to_container_op(my_func)
next_op = components.func_to_container_op(next_func)
@dsl.pipeline(
name='simple pipeline',
description='A trainer that does end-to-end distributed training for XGBoost models.'
)
def my_pipeline():
my_task = my_op()
next_task = next_op("111")
next_task.after(my_task)
if __name__ == '__main__':
compiler = Compiler()
compiler.compile(my_pipeline, "hallo.zip")
# client = kfp.Client()
kfp.Client().create_run_from_pipeline_func(my_pipeline, arguments=None)