def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None):
"""Dump pipeline workflow into yaml spec and write out in the format specified by the user.
Args:
workflow: Workflow spec of the pipline, dict.
package_path: file path to be written. If not specified, a yaml_text string will be returned.
"""
yaml_text = dump_yaml(workflow)
if package_path is None:
return yaml_text
if package_path.endswith('.tar.gz') or package_path.endswith('.tgz'):
from contextlib import closing
from io import BytesIO
with tarfile.open(package_path, "w:gz") as tar:
with closing(BytesIO(yaml_text.encode())) as yaml_file:
tarinfo = tarfile.TarInfo('pipeline.yaml')
tarinfo.size = len(yaml_file.getvalue())
tar.addfile(tarinfo, fileobj=yaml_file)
elif package_path.endswith('.zip'):
with zipfile.ZipFile(package_path, "w") as zip:
zipinfo = zipfile.ZipInfo('pipeline.yaml')
zipinfo.compress_type = zipfile.ZIP_DEFLATED
zip.writestr(zipinfo, yaml_text)
elif package_path.endswith('.yaml') or package_path.endswith('.yml'):
with open(package_path, 'w') as yaml_file:
yaml_file.write(yaml_text)
else:
raise ValueError(
'The output path '+ package_path +
' should ends with one of the following formats: '
'[.tar.gz, .tgz, .zip, .yaml, .yml]')
def save(self, file_path: str):
"""Saves the component definition to file.
It can be shared online and later loaded using the load_component
function.
"""
from kfp.components._yaml_utils import dump_yaml
component_yaml = dump_yaml(self.to_dict())
with open(file_path, 'w') as f:
f.write(component_yaml)
def func_to_component_text(func,
extra_code='',
base_image: str = None,
packages_to_install: List[str] = None,
modules_to_capture: List[str] = None,
use_code_pickling=False):
'''Converts a Python function to a component definition and returns its textual representation.
Function docstring is used as component description. Argument and return
annotations are used as component input/output types.
To declare a function with multiple return values, use the NamedTuple return
annotation syntax::
from typing import NamedTuple
def add_multiply_two_numbers(a: float, b: float) ->
NamedTuple('DummyName', [('sum', float), ('product', float)]):
"""Returns sum and product of two arguments"""
return (a + b, a * b)
Args:
func: The python function to convert
base_image: Optional. Specify a custom Docker container image to use in
the component. For lightweight components, the image needs to have
python 3.5+. Default is python:3.7
extra_code: Optional. Extra code to add before the function code. Can be
used as workaround to define types used in function signature.
packages_to_install: Optional. List of [versioned] python packages to
pip install before executing the user function.
modules_to_capture: Optional. List of module names that will be captured
(instead of just referencing) during the dependency scan. By default
the :code:`func.__module__` is captured. The actual algorithm:
Starting with the initial function, start traversing dependencies.
If the dependency.__module__ is in the modules_to_capture list then
it's captured and it's dependencies are traversed. Otherwise the
dependency is only referenced instead of capturing and its
dependencies are not traversed.
use_code_pickling: Specifies whether the function code should be
captured using pickling as opposed to source code manipulation.
Pickling has better support for capturing dependencies, but is
sensitive to version mismatch between python in component creation
environment and runtime image.
Returns:
Textual representation of a component definition
'''
component_dict = _func_to_component_dict(
func=func,
extra_code=extra_code,
base_image=base_image,
packages_to_install=packages_to_install,
modules_to_capture=modules_to_capture,
use_code_pickling=use_code_pickling,
)
return dump_yaml(component_dict)
def dkube_op(name, token, stage, **kwargs):
assert stage in VALID_STAGES, "Invalid value for stage, must be one of training/preprocessing/serving/custom"
component = None
path = Path(__file__).parent
with open('{}/dkube.yaml'.format(path), 'rb') as stream:
cdict = load_yaml(stream)
cdict['name'] = name
cidct['metadata']['labels']['stage'] = stage
cyaml = dump_yaml(cdict)
component = components.load_component_from_text(cyaml)
assert component != None, "Internal error, loading DKube component failed"
return component(name, token, stage, **kwargs)
def _validate_workflow(workflow: dict):
workflow = workflow.copy()
# Working around Argo lint issue
for argument in workflow['spec'].get('arguments', {}).get('parameters', []):
if 'value' not in argument:
argument['value'] = ''
yaml_text = dump_yaml(workflow)
if '{{pipelineparam' in yaml_text:
raise RuntimeError(
'''Internal compiler error: Found unresolved PipelineParam.
Please create a new issue at https://github.com/kubeflow/pipelines/issues attaching the pipeline code and the pipeline package.'''
)
# Running Argo lint if available
import shutil
import subprocess
argo_path = shutil.which('argo')
if argo_path:
has_working_argo_lint = False
try:
has_working_argo_lint = _run_argo_lint("""
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: hello-world-
spec:
entrypoint: whalesay
templates:
- name: whalesay
container:
image: docker/whalesay:latest""")
except:
warnings.warn("Cannot validate the compiled workflow. Found the argo program in PATH, but it's not usable. argo v2.4.3 should work.")
if has_working_argo_lint:
_run_argo_lint(yaml_text)