def _compile_pipeline_function( pipeline_funcs: List[Callable], function_name: Optional[str], pipeline_parameters: Optional[Mapping[str, Any]], package_path: str, type_check: bool, use_experimental: bool, ) -> None: """Compiles a pipeline function. Args: pipeline_funcs: A list of pipeline_functions. function_name: The name of the pipeline function to compile if there were multiple. pipeline_parameters: The pipeline parameters as a dict of {name: value}. package_path: The output path of the compiled result. type_check: Whether to enable the type checking. """ if len(pipeline_funcs) == 0: raise ValueError( 'A function with @dsl.pipeline decorator is required in the py file.' ) if len(pipeline_funcs) > 1 and not function_name: func_names = [x.__name__ for x in pipeline_funcs] raise ValueError( 'There are multiple pipelines: %s. Please specify --function.' % func_names) if function_name: pipeline_func = next( (x for x in pipeline_funcs if x.__name__ == function_name), None) if not pipeline_func: raise ValueError('The function "%s" does not exist. ' 'Did you forget @dsl.pipeline decoration?' % function_name) else: pipeline_func = pipeline_funcs[0] if use_experimental: experimental_compiler.Compiler().compile( pipeline_func=pipeline_func, pipeline_parameters=pipeline_parameters, package_path=package_path, type_check=type_check) else: compiler.Compiler().compile(pipeline_func=pipeline_func, pipeline_parameters=pipeline_parameters, package_path=package_path, type_check=type_check)
def test_compile_simple_pipeline(self): tmpdir = tempfile.mkdtemp() try: producer_op = components.load_component_from_text(""" name: producer inputs: - {name: input_param, type: String} outputs: - {name: output_model, type: Model} - {name: output_value, type: Integer} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: input_param} - {outputPath: output_model} - {outputPath: output_value} """) consumer_op = components.load_component_from_text(""" name: consumer inputs: - {name: input_model, type: Model} - {name: input_value, type: Integer} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputPath: input_model} - {inputValue: input_value} """) @dsl.pipeline(name='test-pipeline') def simple_pipeline(pipeline_input: str = 'Hello KFP!'): producer = producer_op(input_param=pipeline_input) consumer = consumer_op( input_model=producer.outputs['output_model'], input_value=producer.outputs['output_value']) target_json_file = os.path.join(tmpdir, 'result.json') compiler.Compiler().compile(pipeline_func=simple_pipeline, package_path=target_json_file) self.assertTrue(os.path.exists(target_json_file)) with open(target_json_file, 'r') as f: print(f.read()) finally: shutil.rmtree(tmpdir)
with open(model.path, 'w') as output_file: for i in range(num_steps): output_file.write('Step {}\n{}\n=====\n'.format(i, line)) # model is an instance of Model artifact, which has a .metadata dictionary # to store arbitrary metadata for the output artifact. model.metadata['accuracy'] = 0.9 @dsl.pipeline(pipeline_root='dummy_root', name='my-test-pipeline-beta') def pipeline(message: str, input_dict: Dict[str, int] = {'A': 1, 'B': 2}): preprocess_task = preprocess( message=message, input_dict_parameter=input_dict, input_list_parameter=['a', 'b', 'c'], ) train_task = train( dataset_one_path=preprocess_task.outputs['output_dataset_one'], dataset_two=preprocess_task.outputs['output_dataset_two_path'], message=preprocess_task.outputs['output_parameter_path'], input_bool=preprocess_task.outputs['output_bool_parameter_path'], input_dict=preprocess_task.outputs['output_dict_parameter_path'], input_list=preprocess_task.outputs['output_list_parameter_path'], ) if __name__ == '__main__': compiler.Compiler().compile(pipeline_func=pipeline, package_path=__file__.replace('.py', '.json'))
component_op_2 = components.load_component_from_text(""" name: Read from GCS inputs: - {name: input_gcs_path, type: GCSPath, description: 'GCS file path'} implementation: container: image: google/cloud-sdk:slim command: - sh - -c - | set -e -x gsutil cat "$0" - {inputUri: input_gcs_path} """) @dsl.pipeline(name='simple-two-step-pipeline', pipeline_root='dummy_root') def my_pipeline(text: str = 'Hello world!'): component_1 = component_op_1(text=text).set_display_name('Producer') component_2 = component_op_2( input_gcs_path=component_1.outputs['output_gcs_path']) component_2.set_display_name('Consumer') if __name__ == '__main__': compiler.Compiler().compile(pipeline_func=my_pipeline, pipeline_parameters={'text': 'Hello KFP!'}, package_path=__file__.replace('.py', '.json'))