def _verify_compiled_workflow(self, golden_yaml_file, compiled_workflow):
"""
Tests if the compiled workflow matches the golden yaml.
"""
if GENERATE_GOLDEN_YAML:
# TODO: generate the pipelineloop CRD files akin to ...
# for f in testdata/*_pipelineloop_cr*.yaml; do \
# echo ${f/_pipelineloop_cr*.yaml/.py}; done | sort -u | while read f; do \
# echo $f; dsl-compile-tekton --py $f --output ${f/.py/.yaml}; \
# done
with open(golden_yaml_file, 'w') as f:
f.write(LICENSE_HEADER)
with open(golden_yaml_file, 'a+') as f:
yaml_text = dump_yaml(compiled_workflow)
f.write(yaml_text)
else:
with open(golden_yaml_file, 'r') as f:
golden = yaml.safe_load(f)
self.maxDiff = None
# sort dicts and lists, insertion order was not guaranteed before Python 3.6
if sys.version_info < (3, 6, 0):
def sort_items(obj):
from collections import OrderedDict
if isinstance(obj, dict):
return OrderedDict(
{k: sort_items(v)
for k, v in sorted(obj.items())})
elif isinstance(obj, list):
return sorted([sort_items(o) for o in obj],
key=lambda x: str(x))
else:
return obj
golden = sort_items(golden)
compiled_workflow = sort_items(compiled_workflow)
self.assertEqual(
golden,
compiled_workflow,
msg="\n===[ " + golden_yaml_file.split(os.path.sep)[-1] +
" ]===\n" + json.dumps(compiled_workflow, indent=2))
def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None):
"""Dump pipeline workflow into yaml spec and write out in the format specified by the user.
Args:
workflow: Workflow spec of the pipline, dict.
package_path: file path to be written. If not specified, a yaml_text string
will be returned.
"""
yaml_text = dump_yaml(workflow)
# Use regex to replace all the Argo variables to Tekton variables. For variables that are unique to Argo,
# we raise an Error to alert users about the unsupported variables. Here is the list of Argo variables.
# https://github.com/argoproj/argo/blob/master/docs/variables.md
# Since Argo variables can be used in anywhere in the yaml, we need to dump and then parse the whole yaml
# using regular expression.
tekton_var_regex_rules = [{
'argo_rule': '{{inputs.parameters.([^ \t\n.:,;{}]+)}}',
'tekton_rule': '$(inputs.params.\g<1>)'
}, {
'argo_rule': '{{outputs.parameters.([^ \t\n.:,;{}]+).path}}',
'tekton_rule': '$(results.\g<1>.path)'
}, {
'argo_rule': '{{workflow.uid}}',
'tekton_rule': '$(context.pipelineRun.uid)'
}, {
'argo_rule': '{{workflow.name}}',
'tekton_rule': '$(context.pipelineRun.name)'
}, {
'argo_rule':
'{{workflow.namespace}}',
'tekton_rule':
'$(context.pipelineRun.namespace)'
}, {
'argo_rule': '{{workflow.parameters.([^ \t\n.:,;{}]+)}}',
'tekton_rule': '$(params.\g<1>)'
}]
for regex_rule in tekton_var_regex_rules:
yaml_text = re.sub(regex_rule['argo_rule'],
regex_rule['tekton_rule'], yaml_text)
unsupported_vars = re.findall(r"{{[^ \t\n.:,;{}]+\.[^ \t\n:,;{}]+}}",
yaml_text)
if unsupported_vars:
raise ValueError(
'These Argo variables are not supported in Tekton Pipeline: %s'
% ", ".join(str(v) for v in set(unsupported_vars)))
if '{{pipelineparam' in yaml_text:
raise RuntimeError(
'Internal compiler error: Found unresolved PipelineParam. '
'Please create a new issue at https://github.com/kubeflow/kfp-tekton/issues '
'attaching the pipeline DSL code and the pipeline YAML.')
yaml_text = dump_yaml(
_handle_tekton_pipeline_variables(
yaml.load(yaml_text, Loader=yaml.FullLoader)))
if package_path is None:
return yaml_text
if package_path.endswith('.tar.gz') or package_path.endswith('.tgz'):
from contextlib import closing
from io import BytesIO
with tarfile.open(package_path, "w:gz") as tar:
with closing(BytesIO(yaml_text.encode())) as yaml_file:
tarinfo = tarfile.TarInfo('pipeline.yaml')
tarinfo.size = len(yaml_file.getvalue())
tar.addfile(tarinfo, fileobj=yaml_file)
elif package_path.endswith('.zip'):
with zipfile.ZipFile(package_path, "w") as zip:
zipinfo = zipfile.ZipInfo('pipeline.yaml')
zipinfo.compress_type = zipfile.ZIP_DEFLATED
zip.writestr(zipinfo, yaml_text)
elif package_path.endswith('.yaml') or package_path.endswith('.yml'):
with open(package_path, 'w') as yaml_file:
yaml_file.write(yaml_text)
else:
raise ValueError(
'The output path %s should end with one of the following formats: '
'[.tar.gz, .tgz, .zip, .yaml, .yml]' % package_path)