def _assert_compiled_pipeline_equals_golden( self, kfp_compiler: compiler.Compiler, pipeline_func: Callable, golden_yaml_filename: str): compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml') kfp_compiler.compile(pipeline_func, package_path=compiled_file) test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_file = os.path.join(test_data_dir, golden_yaml_filename) # Uncomment the following to update goldens. # TODO: place this behind some --update_goldens flag. # kfp_compiler.compile(pipeline_func, package_path=golden_file) with open(golden_file, 'r') as f: golden = yaml.safe_load(f) with open(compiled_file, 'r') as f: compiled = yaml.safe_load(f) for workflow in golden, compiled: del workflow['metadata'] for template in workflow['spec']['templates']: template.pop('metadata', None) self.maxDiff = None self.assertDictEqual(golden, compiled)
def _assert_compiled_pipeline_equals_golden(self, kfp_compiler: compiler.Compiler, pipeline_func: Callable, golden_yaml_filename: str): compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml') kfp_compiler.compile(pipeline_func, package_path=compiled_file) test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_file = os.path.join(test_data_dir, golden_yaml_filename) # Uncomment the following to update goldens. # TODO: place this behind some --update_goldens flag. # kfp_compiler.compile(pipeline_func, package_path=golden_file) with open(golden_file, 'r') as f: golden = yaml.safe_load(f) with open(compiled_file, 'r') as f: compiled = yaml.safe_load(f) for workflow in golden, compiled: del workflow['metadata'] for template in workflow['spec']['templates']: template.pop('metadata', None) if 'initContainers' not in template: continue # Strip off the launcher image label before comparison for initContainer in template['initContainers']: initContainer['image'] = initContainer['image'].split(':')[0] self.maxDiff = None self.assertDictEqual(golden, compiled)
def _assert_compiled_pipeline_equals_golden(self, kfp_compiler: compiler.Compiler, pipeline_func: Callable, golden_yaml_filename: str): compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml') kfp_compiler.compile(pipeline_func, package_path=compiled_file) test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') golden_file = os.path.join(test_data_dir, golden_yaml_filename) def _load_compiled_template(filename: str) -> Dict: with open(filename, 'r') as f: workflow = yaml.safe_load(f) del workflow['metadata'] for template in workflow['spec']['templates']: template.pop('metadata', None) if 'initContainers' not in template: continue # Strip off the launcher image label before comparison for initContainer in template['initContainers']: initContainer['image'] = initContainer['image'].split( ':')[0] if 'container' in template: template['container'] = json.loads( re.sub("'kfp==(\d+).(\d+).(\d+)'", 'kfp', json.dumps(template['container']))) return workflow golden = _load_compiled_template(golden_file) compiled = _load_compiled_template(compiled_file) # Devs can run the following command to update golden files: # UPDATE_GOLDENS=True python3 -m unittest kfp/compiler/v2_compatible_compiler_test.py # If UPDATE_GOLDENS=True, and the diff is # different, update the golden file and reload it. update_goldens = os.environ.get('UPDATE_GOLDENS', False) if golden != compiled and update_goldens: kfp_compiler.compile(pipeline_func, package_path=golden_file) golden = _load_compiled_template(golden_file) self.assertDictEqual(golden, compiled)
import kfp from kfp import dsl from kfp.compiler import Compiler #sb_op = kfp.components.load_component_from_url("https://raw.githubusercontent.com/demotto/my-pipeline/master/expr001/component.yaml") import requests url = "https://raw.githubusercontent.com/demotto/my-pipeline/master/expr001/component.yaml" resp = requests.get(url) sb_op = kfp.components.load_component_from_text(resp.text) @dsl.pipeline( name='simple pipeline', description='A trainer that does end-to-end distributed training for XGBoost models.' ) def my_pipeline(): my_task = sb_op( input1="xxxxxxxxxx", input2="oooooooooo" ) if __name__ == '__main__': compiler = Compiler() compiler.compile(my_pipeline, "my_brick.yaml")
import lady_gaga def my_func(): lady_gaga.my_func_impl() def next_func(a): lady_gaga.next_func_impl() my_op = components.func_to_container_op(my_func) next_op = components.func_to_container_op(next_func) @dsl.pipeline( name='simple pipeline', description='A trainer that does end-to-end distributed training for XGBoost models.' ) def my_pipeline(): my_task = my_op() next_task = next_op("111") next_task.after(my_task) if __name__ == '__main__': compiler = Compiler() compiler.compile(my_pipeline, "hallo.zip") # client = kfp.Client() kfp.Client().create_run_from_pipeline_func(my_pipeline, arguments=None)