def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None): """Dump pipeline workflow into yaml spec and write out in the format specified by the user. Args: workflow: Workflow spec of the pipline, dict. package_path: file path to be written. If not specified, a yaml_text string will be returned. """ yaml_text = dump_yaml(workflow) if package_path is None: return yaml_text if package_path.endswith('.tar.gz') or package_path.endswith('.tgz'): from contextlib import closing from io import BytesIO with tarfile.open(package_path, "w:gz") as tar: with closing(BytesIO(yaml_text.encode())) as yaml_file: tarinfo = tarfile.TarInfo('pipeline.yaml') tarinfo.size = len(yaml_file.getvalue()) tar.addfile(tarinfo, fileobj=yaml_file) elif package_path.endswith('.zip'): with zipfile.ZipFile(package_path, "w") as zip: zipinfo = zipfile.ZipInfo('pipeline.yaml') zipinfo.compress_type = zipfile.ZIP_DEFLATED zip.writestr(zipinfo, yaml_text) elif package_path.endswith('.yaml') or package_path.endswith('.yml'): with open(package_path, 'w') as yaml_file: yaml_file.write(yaml_text) else: raise ValueError( 'The output path '+ package_path + ' should ends with one of the following formats: ' '[.tar.gz, .tgz, .zip, .yaml, .yml]')
def save(self, file_path: str): """Saves the component definition to file. It can be shared online and later loaded using the load_component function. """ from kfp.components._yaml_utils import dump_yaml component_yaml = dump_yaml(self.to_dict()) with open(file_path, 'w') as f: f.write(component_yaml)
def func_to_component_text(func, extra_code='', base_image: str = None, packages_to_install: List[str] = None, modules_to_capture: List[str] = None, use_code_pickling=False): '''Converts a Python function to a component definition and returns its textual representation. Function docstring is used as component description. Argument and return annotations are used as component input/output types. To declare a function with multiple return values, use the NamedTuple return annotation syntax:: from typing import NamedTuple def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): """Returns sum and product of two arguments""" return (a + b, a * b) Args: func: The python function to convert base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is python:3.7 extra_code: Optional. Extra code to add before the function code. Can be used as workaround to define types used in function signature. packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. modules_to_capture: Optional. List of module names that will be captured (instead of just referencing) during the dependency scan. By default the :code:`func.__module__` is captured. The actual algorithm: Starting with the initial function, start traversing dependencies. If the dependency.__module__ is in the modules_to_capture list then it's captured and it's dependencies are traversed. Otherwise the dependency is only referenced instead of capturing and its dependencies are not traversed. use_code_pickling: Specifies whether the function code should be captured using pickling as opposed to source code manipulation. Pickling has better support for capturing dependencies, but is sensitive to version mismatch between python in component creation environment and runtime image. Returns: Textual representation of a component definition ''' component_dict = _func_to_component_dict( func=func, extra_code=extra_code, base_image=base_image, packages_to_install=packages_to_install, modules_to_capture=modules_to_capture, use_code_pickling=use_code_pickling, ) return dump_yaml(component_dict)
def dkube_op(name, token, stage, **kwargs): assert stage in VALID_STAGES, "Invalid value for stage, must be one of training/preprocessing/serving/custom" component = None path = Path(__file__).parent with open('{}/dkube.yaml'.format(path), 'rb') as stream: cdict = load_yaml(stream) cdict['name'] = name cidct['metadata']['labels']['stage'] = stage cyaml = dump_yaml(cdict) component = components.load_component_from_text(cyaml) assert component != None, "Internal error, loading DKube component failed" return component(name, token, stage, **kwargs)
def _validate_workflow(workflow: dict): workflow = workflow.copy() # Working around Argo lint issue for argument in workflow['spec'].get('arguments', {}).get('parameters', []): if 'value' not in argument: argument['value'] = '' yaml_text = dump_yaml(workflow) if '{{pipelineparam' in yaml_text: raise RuntimeError( '''Internal compiler error: Found unresolved PipelineParam. Please create a new issue at https://github.com/kubeflow/pipelines/issues attaching the pipeline code and the pipeline package.''' ) # Running Argo lint if available import shutil import subprocess argo_path = shutil.which('argo') if argo_path: has_working_argo_lint = False try: has_working_argo_lint = _run_argo_lint(""" apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hello-world- spec: entrypoint: whalesay templates: - name: whalesay container: image: docker/whalesay:latest""") except: warnings.warn("Cannot validate the compiled workflow. Found the argo program in PATH, but it's not usable. argo v2.4.3 should work.") if has_working_argo_lint: _run_argo_lint(yaml_text)