def _op_to_template(op: BaseOp,
pipelinerun_output_artifacts={},
enable_artifacts=False):
"""Generate template given an operator inherited from BaseOp."""
# initial local variables for tracking volumes and artifacts
volume_mount_step_template = []
volume_template = []
mounted_param_paths = []
replaced_param_list = []
artifact_to_result_mapping = {}
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
# print(op.output_artifact_paths)
# This should have been as easy as output_artifact_paths.update(op.file_outputs),
# but the _outputs_to_json function changes the output names and we must do the same here,
# so that the names are the same
output_artifact_paths.update(
sorted(((param.full_name, processed_op.file_outputs[param.name])
for param in processed_op.outputs.values()),
key=lambda x: x[0]))
output_artifacts = [
convert_k8s_obj_to_json(
ArtifactLocation.create_artifact_for_s3(
op.artifact_location,
name=name,
path=path,
key='runs/$PIPELINERUN/$PIPELINETASK/' + name))
for name, path in output_artifact_paths.items()
] if enable_artifacts else []
# workflow template
container = convert_k8s_obj_to_json(processed_op.container)
# Calling containerOp step as "main" to align with Argo
step = {'name': "main"}
step.update(container)
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
'steps': [step]
}
}
# Create output artifact tracking annotation.
if enable_artifacts:
for output_artifact in output_artifacts:
output_annotation = pipelinerun_output_artifacts.get(
processed_op.name, [])
output_annotation.append({
'name': output_artifact['name'],
'path': output_artifact['path']
})
pipelinerun_output_artifacts[
processed_op.name] = output_annotation
elif isinstance(op, dsl.ResourceOp):
# no output artifacts
output_artifacts = []
# Flatten manifest because it needs to replace Argo variables
manifest = yaml.dump(convert_k8s_obj_to_json(
processed_op.k8s_resource),
default_flow_style=False)
argo_var = False
if manifest.find('{{workflow.name}}') != -1:
# Kubernetes Pod arguments only take $() as environment variables
manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)")
# Remove yaml quote in order to read bash variables
manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest)
argo_var = True
# task template
template = _get_resourceOp_template(op,
processed_op.name,
tekton_api_version,
manifest,
argo_var=argo_var)
# initContainers
if processed_op.init_containers:
template['spec']['steps'] = _prepend_steps(
processed_op.init_containers, template['spec']['steps'])
# inputs
input_artifact_paths = processed_op.input_artifact_paths if isinstance(
processed_op, dsl.ContainerOp) else None
artifact_arguments = processed_op.artifact_arguments if isinstance(
processed_op, dsl.ContainerOp) else None
inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths,
artifact_arguments)
if 'parameters' in inputs:
if isinstance(processed_op, dsl.ContainerOp):
template['spec']['params'] = inputs['parameters']
elif isinstance(op, dsl.ResourceOp):
template['spec']['params'].extend(inputs['parameters'])
if 'artifacts' in inputs:
# Leave artifacts for big data passing
template['spec']['artifacts'] = inputs['artifacts']
# outputs
if isinstance(op, dsl.ContainerOp):
op_outputs = processed_op.outputs
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
op_outputs = {}
param_outputs = {}
outputs_dict = _outputs_to_json(op, op_outputs, param_outputs,
output_artifacts)
if outputs_dict:
copy_results_step = _process_parameters(
processed_op, template, outputs_dict, volume_mount_step_template,
volume_template, replaced_param_list, artifact_to_result_mapping,
mounted_param_paths)
copy_artifacts_step = _process_output_artifacts(
outputs_dict, volume_mount_step_template, volume_template,
replaced_param_list, artifact_to_result_mapping)
if mounted_param_paths:
template['spec']['steps'].append(copy_results_step)
_update_volumes(template, volume_mount_step_template, volume_template)
if copy_artifacts_step:
template['spec']['steps'].append(copy_artifacts_step)
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template.setdefault(
'metadata', {}) # Tekton change, don't wipe out existing metadata
if processed_op.pod_annotations:
template['metadata']['annotations'] = {
sanitize_k8s_name(key,
allow_capital_underscore=True,
allow_dot=True,
allow_slash=True,
max_length=253): value
for key, value in processed_op.pod_annotations.items()
}
if processed_op.pod_labels:
template['metadata']['labels'] = {
sanitize_k8s_name(key,
allow_capital_underscore=True,
allow_dot=True,
allow_slash=True,
max_length=253):
sanitize_k8s_name(value,
allow_capital_underscore=True,
allow_dot=True)
for key, value in processed_op.pod_labels.items()
}
# sidecars
if processed_op.sidecars:
template['spec']['sidecars'] = processed_op.sidecars
# volumes
if processed_op.volumes:
template['spec']['volumes'] = template['spec'].get('volume', []) + [
convert_k8s_obj_to_json(volume) for volume in processed_op.volumes
]
template['spec']['volumes'].sort(key=lambda x: x['name'])
# Display name
if processed_op.display_name:
template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/task_display_name'] = \
processed_op.display_name
if isinstance(op, dsl.ContainerOp) and op._metadata:
template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/component_spec'] = \
json.dumps(op._metadata.to_dict(), sort_keys=True)
if isinstance(op, dsl.ContainerOp) and op.execution_options:
if op.execution_options.caching_strategy.max_cache_staleness:
template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/max_cache_staleness'] = \
str(op.execution_options.caching_strategy.max_cache_staleness)
return template
def _op_to_template(op: BaseOp):
"""Generate template given an operator inherited from BaseOp."""
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
# print(op.output_artifact_paths)
# This should have been as easy as output_artifact_paths.update(op.file_outputs), but the _outputs_to_json function changes the output names and we must do the same here, so that the names are the same
output_artifact_paths.update(
sorted(((param.full_name, processed_op.file_outputs[param.name])
for param in processed_op.outputs.values()),
key=lambda x: x[0]))
output_artifacts = [
# convert_k8s_obj_to_json(
# ArtifactLocation.create_artifact_for_s3(
# op.artifact_location,
# name=name,
# path=path,
# key='runs/{{workflow.uid}}/{{pod.name}}/' + name + '.tgz'))
# for name, path in output_artifact_paths.items()
]
# workflow template
container = convert_k8s_obj_to_json(processed_op.container)
step = {'name': processed_op.name}
step.update(container)
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
'steps': [step]
}
}
elif isinstance(op, dsl.ResourceOp):
# # no output artifacts
# output_artifacts = []
#
# # workflow template
# processed_op.resource["manifest"] = yaml.dump(
# convert_k8s_obj_to_json(processed_op.k8s_resource),
# default_flow_style=False
# )
# template = {
# 'name': processed_op.name,
# 'resource': convert_k8s_obj_to_json(
# processed_op.resource
# )
# }
raise NotImplementedError("dsl.ResourceOp is not yet implemented")
# initContainers
if processed_op.init_containers:
steps = processed_op.init_containers.copy()
steps.extend(template['spec']['steps'])
template['spec']['steps'] = steps
# inputs
input_artifact_paths = processed_op.input_artifact_paths if isinstance(
processed_op, dsl.ContainerOp) else None
artifact_arguments = processed_op.artifact_arguments if isinstance(
processed_op, dsl.ContainerOp) else None
inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths,
artifact_arguments)
if 'parameters' in inputs:
template['spec']['params'] = inputs['parameters']
elif 'artifacts' in inputs:
raise NotImplementedError("input artifacts are not yet implemented")
# outputs
if isinstance(op, dsl.ContainerOp):
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
param_outputs = processed_op.attribute_outputs
outputs_dict = _outputs_to_json(op, processed_op.outputs, param_outputs,
output_artifacts)
if outputs_dict:
template['spec']['results'] = []
for name, path in processed_op.file_outputs.items():
name = name.replace(
'_', '-'
) # replace '_' to '-' since tekton results doesn't support underscore
template['spec']['results'].append({
'name': name,
'description': path
})
# replace all occurrences of the output file path with the Tekton output parameter expression
for s in template['spec']['steps']:
if 'command' in s:
s['command'] = [
c.replace(path, '$(results.%s.path)' % name)
for c in s['command']
]
if 'args' in s:
s['args'] = [
a.replace(path, '$(results.%s.path)' % name)
for a in s['args']
]
# **********************************************************
# NOTE: the following features are still under development
# **********************************************************
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template.setdefault(
'metadata', {}) # Tekton change, don't wipe out existing metadata
if processed_op.pod_annotations:
template['metadata']['annotations'] = processed_op.pod_annotations
if processed_op.pod_labels:
template['metadata']['labels'] = processed_op.pod_labels
# sidecars
if processed_op.sidecars:
template['spec']['sidecars'] = processed_op.sidecars
# volumes
if processed_op.volumes:
template['spec']['volumes'] = [
convert_k8s_obj_to_json(volume) for volume in processed_op.volumes
]
template['spec']['volumes'].sort(key=lambda x: x['name'])
# Display name
if processed_op.display_name:
template.setdefault('metadata', {}).setdefault(
'annotations', {}
)['pipelines.kubeflow.org/task_display_name'] = processed_op.display_name
if isinstance(op, dsl.ContainerOp) and op._metadata:
import json
template.setdefault('metadata', {}).setdefault(
'annotations',
{})['pipelines.kubeflow.org/component_spec'] = json.dumps(
op._metadata.to_dict(), sort_keys=True)
return template
def _op_to_template(op: BaseOp, enable_artifacts=False):
"""Generate template given an operator inherited from BaseOp."""
# initial local variables for tracking volumes and artifacts
volume_mount_step_template = []
volume_template = []
mounted_param_paths = []
replaced_param_list = []
artifact_to_result_mapping = {}
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
# print(op.output_artifact_paths)
# This should have been as easy as output_artifact_paths.update(op.file_outputs),
# but the _outputs_to_json function changes the output names and we must do the same here,
# so that the names are the same
output_artifact_paths.update(
sorted(((param.full_name, processed_op.file_outputs[param.name])
for param in processed_op.outputs.values()),
key=lambda x: x[0]))
output_artifacts = [
convert_k8s_obj_to_json(
ArtifactLocation.create_artifact_for_s3(
op.artifact_location,
name=name,
path=path,
key='runs/$PIPELINERUN/$PODNAME/' + name))
for name, path in output_artifact_paths.items()
] if enable_artifacts else []
# workflow template
container = convert_k8s_obj_to_json(processed_op.container)
step = {'name': processed_op.name}
step.update(container)
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
'steps': [step]
}
}
elif isinstance(op, dsl.ResourceOp):
# no output artifacts
output_artifacts = []
# Flatten manifest because it needs to replace Argo variables
manifest = yaml.dump(convert_k8s_obj_to_json(
processed_op.k8s_resource),
default_flow_style=False)
argo_var = False
if manifest.find('{{workflow.name}}') != -1:
# Kubernetes Pod arguments only take $() as environment variables
manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)")
# Remove yaml quote in order to read bash variables
manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest)
argo_var = True
# task template
template = _get_resourceOp_template(op,
processed_op.name,
tekton_api_version,
manifest,
argo_var=argo_var)
# initContainers
if processed_op.init_containers:
template['spec']['steps'] = _prepend_steps(
processed_op.init_containers, template['spec']['steps'])
# inputs
input_artifact_paths = processed_op.input_artifact_paths if isinstance(
processed_op, dsl.ContainerOp) else None
artifact_arguments = processed_op.artifact_arguments if isinstance(
processed_op, dsl.ContainerOp) else None
inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths,
artifact_arguments)
if 'parameters' in inputs:
if isinstance(processed_op, dsl.ContainerOp):
template['spec']['params'] = inputs['parameters']
elif isinstance(op, dsl.ResourceOp):
template['spec']['params'].extend(inputs['parameters'])
if 'artifacts' in inputs:
# The input artifacts in KFP is not pulling from s3, it will always be passed as a raw input.
# Visit https://github.com/kubeflow/pipelines/issues/336 for more details on the implementation.
copy_inputs_step = _get_base_step('copy-inputs')
for artifact in inputs['artifacts']:
if 'raw' in artifact:
copy_inputs_step['script'] += 'echo -n "%s" > %s\n' % (
artifact['raw']['data'], artifact['path'])
mount_path = artifact['path'].rsplit("/", 1)[0]
if mount_path not in mounted_param_paths:
_add_mount_path(artifact['name'], artifact['path'], mount_path,
volume_mount_step_template, volume_template,
mounted_param_paths)
template['spec']['steps'] = _prepend_steps([copy_inputs_step],
template['spec']['steps'])
_update_volumes(template, volume_mount_step_template, volume_template)
# outputs
if isinstance(op, dsl.ContainerOp):
op_outputs = processed_op.outputs
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
op_outputs = {}
param_outputs = {}
outputs_dict = _outputs_to_json(op, op_outputs, param_outputs,
output_artifacts)
if outputs_dict:
copy_results_step = _process_parameters(
processed_op, template, outputs_dict, volume_mount_step_template,
volume_template, replaced_param_list, artifact_to_result_mapping,
mounted_param_paths)
copy_artifacts_step = _process_output_artifacts(
outputs_dict, volume_mount_step_template, volume_template,
replaced_param_list, artifact_to_result_mapping)
if mounted_param_paths:
template['spec']['steps'].append(copy_results_step)
_update_volumes(template, volume_mount_step_template, volume_template)
if copy_artifacts_step:
template['spec']['steps'].append(copy_artifacts_step)
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template.setdefault(
'metadata', {}) # Tekton change, don't wipe out existing metadata
if processed_op.pod_annotations:
template['metadata']['annotations'] = processed_op.pod_annotations
if processed_op.pod_labels:
template['metadata']['labels'] = processed_op.pod_labels
# sidecars
if processed_op.sidecars:
template['spec']['sidecars'] = processed_op.sidecars
# volumes
if processed_op.volumes:
template['spec']['volumes'] = template['spec'].get('volume', []) + [
convert_k8s_obj_to_json(volume) for volume in processed_op.volumes
]
template['spec']['volumes'].sort(key=lambda x: x['name'])
# Display name
if processed_op.display_name:
template.setdefault('metadata', {}).setdefault(
'annotations', {}
)['pipelines.kubeflow.org/task_display_name'] = processed_op.display_name
if isinstance(op, dsl.ContainerOp) and op._metadata:
template.setdefault('metadata', {}).setdefault(
'annotations',
{})['pipelines.kubeflow.org/component_spec'] = json.dumps(
op._metadata.to_dict(), sort_keys=True)
if isinstance(op, dsl.ContainerOp) and op.execution_options:
if op.execution_options.caching_strategy.max_cache_staleness:
template.setdefault('metadata', {}).setdefault(
'annotations',
{})['pipelines.kubeflow.org/max_cache_staleness'] = str(
op.execution_options.caching_strategy.max_cache_staleness)
return template
def _op_to_template(op: BaseOp,
pipelinerun_output_artifacts={},
artifact_items={}):
"""Generate template given an operator inherited from BaseOp."""
# Display name
if op.display_name:
op.add_pod_annotation('pipelines.kubeflow.org/task_display_name',
op.display_name)
# initial local variables for tracking volumes and artifacts
volume_mount_step_template = []
volume_template = []
mounted_param_paths = []
replaced_param_list = []
artifact_to_result_mapping = {}
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
# print(op.output_artifact_paths)
# This should have been as easy as output_artifact_paths.update(op.file_outputs),
# but the _outputs_to_json function changes the output names and we must do the same here,
# so that the names are the same
output_artifact_paths.update(
sorted(((param.full_name, processed_op.file_outputs[param.name])
for param in processed_op.outputs.values()),
key=lambda x: x[0]))
output_artifacts = [{
'name': name,
'path': path
} for name, path in output_artifact_paths.items()]
# workflow template
container = convert_k8s_obj_to_json(processed_op.container)
# Calling containerOp step as "main" to align with Argo
step = {'name': "main"}
step.update(container)
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
'steps': [step]
}
}
# Create output artifact tracking annotation.
for output_artifact in output_artifacts:
output_annotation = pipelinerun_output_artifacts.get(
processed_op.name, [])
output_annotation.append({
'name':
output_artifact.get('name', ''),
'path':
output_artifact.get('path', ''),
'key':
"artifacts/$PIPELINERUN/%s/%s.tgz" %
(processed_op.name, output_artifact.get('name', '').replace(
processed_op.name + '-', ''))
})
pipelinerun_output_artifacts[processed_op.name] = output_annotation
elif isinstance(op, dsl.ResourceOp):
# no output artifacts
output_artifacts = []
# Flatten manifest because it needs to replace Argo variables
manifest = yaml.dump(convert_k8s_obj_to_json(
processed_op.k8s_resource),
default_flow_style=False)
argo_var = False
if manifest.find('{{workflow.name}}') != -1:
# Kubernetes Pod arguments only take $() as environment variables
manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)")
# Remove yaml quote in order to read bash variables
manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest)
argo_var = True
# task template
template = _get_resourceOp_template(op,
processed_op.name,
tekton_api_version,
manifest,
argo_var=argo_var)
# initContainers
if processed_op.init_containers:
template['spec']['steps'] = _prepend_steps(
processed_op.init_containers, template['spec']['steps'])
# inputs
input_artifact_paths = processed_op.input_artifact_paths if isinstance(
processed_op, dsl.ContainerOp) else None
artifact_arguments = processed_op.artifact_arguments if isinstance(
processed_op, dsl.ContainerOp) else None
inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths,
artifact_arguments)
if 'parameters' in inputs:
if isinstance(processed_op, dsl.ContainerOp):
template['spec']['params'] = inputs['parameters']
elif isinstance(op, dsl.ResourceOp):
template['spec']['params'].extend(inputs['parameters'])
if 'artifacts' in inputs:
# Leave artifacts for big data passing
template['spec']['artifacts'] = inputs['artifacts']
# outputs
if isinstance(op, dsl.ContainerOp):
op_outputs = processed_op.outputs
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
op_outputs = {}
param_outputs = {}
outputs_dict = _outputs_to_json(op, op_outputs, param_outputs,
output_artifacts)
artifact_items[op.name] = artifact_items.get(op.name, [])
if outputs_dict:
copy_results_step = _process_parameters(
processed_op, template, outputs_dict, volume_mount_step_template,
volume_template, replaced_param_list, artifact_to_result_mapping,
mounted_param_paths)
_process_output_artifacts(outputs_dict, volume_mount_step_template,
volume_template, replaced_param_list,
artifact_to_result_mapping,
artifact_items[op.name])
if mounted_param_paths:
template['spec']['steps'].append(copy_results_step)
_update_volumes(template, volume_mount_step_template, volume_template)
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template.setdefault(
'metadata', {}) # Tekton change, don't wipe out existing metadata
if processed_op.pod_annotations:
template['metadata']['annotations'] = {
sanitize_k8s_name(key,
allow_capital_underscore=True,
allow_dot=True,
allow_slash=True,
max_length=253): value
for key, value in processed_op.pod_annotations.items()
}
if processed_op.pod_labels:
template['metadata']['labels'] = {
sanitize_k8s_name(key,
allow_capital_underscore=True,
allow_dot=True,
allow_slash=True,
max_length=253):
sanitize_k8s_name(value,
allow_capital_underscore=True,
allow_dot=True)
for key, value in processed_op.pod_labels.items()
}
# sidecars
if processed_op.sidecars:
template['spec']['sidecars'] = processed_op.sidecars
# volumes
if processed_op.volumes:
template['spec']['volumes'] = template['spec'].get('volumes', []) + [
convert_k8s_obj_to_json(volume) for volume in processed_op.volumes
]
template['spec']['volumes'].sort(key=lambda x: x['name'])
if isinstance(op, dsl.ContainerOp
) and op._metadata and GENERATE_COMPONENT_SPEC_ANNOTATIONS:
component_spec_dict = op._metadata.to_dict()
component_spec_digest = hashlib.sha256(
json.dumps(component_spec_dict,
sort_keys=True).encode()).hexdigest()
component_name = component_spec_dict.get('name', op.name)
component_version = component_name + '@sha256=' + component_spec_digest
digested_component_spec_dict = {
'name': component_name,
'outputs': component_spec_dict.get('outputs', []),
'version': component_version
}
template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/component_spec_digest'] = \
json.dumps(digested_component_spec_dict, sort_keys=True)
if isinstance(op, dsl.ContainerOp) and op.execution_options:
if op.execution_options.caching_strategy.max_cache_staleness:
template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/max_cache_staleness'] = \
str(op.execution_options.caching_strategy.max_cache_staleness)
# Sort and arrange results based on provided estimate size and process results in multi-steps if the result sizes are too big.
result_size_map = "{}"
if processed_op.pod_annotations:
result_size_map = processed_op.pod_annotations.get(
"tekton-result-sizes", "{}")
# Only sort and arrange results when the estimated sizes are given.
if result_size_map and result_size_map != "{}":
try:
result_size_map = json.loads(result_size_map)
except ValueError:
raise ("tekton-result-sizes annotation is not a valid JSON")
# Normalize estimated result size keys.
result_size_map = {
sanitize_k8s_name(key, allow_capital_underscore=True): value
for key, value in result_size_map.items()
}
# Sort key orders based on values
result_size_map = dict(
sorted(result_size_map.items(),
key=lambda item: item[1],
reverse=True))
max_byte_size = 2048
verified_result_size_map = {0: {}}
op_result_names = [
name['name'] for name in template['spec']['results']
]
step_bins = {0: 0}
step_counter = 0
# Group result files to not exceed max_byte_size as a bin packing problem
# Results are sorted from large to small, each value will loop over each bin to determine can it fit in the existing bins.
for key, value in result_size_map.items():
try:
value = int(value)
except ValueError:
raise (
"Estimated value for result %s is %s, but it needs to be an integer."
% (key, value))
if key in op_result_names:
packed_index = -1
# Look for bin that can fit the result value
for i in range(len(step_bins)):
if step_bins[i] + value > max_byte_size:
continue
step_bins[i] = step_bins[i] + value
packed_index = i
break
# If no bin can fit the value, create a new bin to store the value
if packed_index < 0:
step_counter += 1
if value > max_byte_size:
logging.warning(
"The estimated size for parameter %s is %sB which is more than 2KB, "
"consider passing this value as artifact instead of output parameter."
% (key, str(value)))
step_bins[step_counter] = value
verified_result_size_map[step_counter] = {}
packed_index = step_counter
verified_result_size_map[packed_index][key] = value
else:
logging.warning(
"The esitmated size for parameter %s does not exist in the task %s."
"Please correct the task annotations with the correct parameter key"
% (key, op.name))
missing_param_estimation = []
for result_name in op_result_names:
if result_name not in result_size_map.keys():
missing_param_estimation.append(result_name)
if missing_param_estimation:
logging.warning(
"The following output parameter estimations are missing in task %s: Missing params: %s."
% (op.name, missing_param_estimation))
# Move results between the Tekton home and result directories if there are more than one step
if step_counter > 0:
for step in template['spec']['steps']:
if step['name'] == 'main':
for key in result_size_map.keys():
# Replace main step results that are not in the first bin to the Tekton home path
if key not in verified_result_size_map[0].keys():
sanitize_key = sanitize_k8s_name(key)
for i, a in enumerate(step['args']):
a = a.replace(
'$(results.%s.path)' % sanitize_key,
'%s%s' %
(TEKTON_HOME_RESULT_PATH, sanitize_key))
step['args'][i] = a
for i, c in enumerate(step['command']):
c = c.replace(
'$(results.%s.path)' % sanitize_key,
'%s%s' %
(TEKTON_HOME_RESULT_PATH, sanitize_key))
step['command'][i] = c
# Append new steps to move result files between each step, so Tekton controller can record all results without
# exceeding the Kubernetes termination log limit.
for i in range(1, step_counter + 1):
copy_result_step = _get_copy_result_step_template(
i, verified_result_size_map)
template['spec']['steps'].append(copy_result_step)
# Update actifact item location to the latest stage in order to properly track and store all the artifacts.
for i, artifact in enumerate(artifact_items[op.name]):
if artifact[0] not in verified_result_size_map[step_counter].keys(
):
artifact[1] = '%s%s' % (TEKTON_HOME_RESULT_PATH,
sanitize_k8s_name(artifact[0]))
artifact_items[op.name][i] = artifact
return template
def _op_to_template(op: BaseOp):
"""Generate template given an operator inherited from BaseOp."""
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
# print(op.output_artifact_paths)
# This should have been as easy as output_artifact_paths.update(op.file_outputs), but the _outputs_to_json function changes the output names and we must do the same here, so that the names are the same
output_artifact_paths.update(
sorted(((param.full_name, processed_op.file_outputs[param.name])
for param in processed_op.outputs.values()),
key=lambda x: x[0]))
output_artifacts = [
# convert_k8s_obj_to_json(
# ArtifactLocation.create_artifact_for_s3(
# op.artifact_location,
# name=name,
# path=path,
# key='runs/{{workflow.uid}}/{{pod.name}}/' + name + '.tgz'))
# for name, path in output_artifact_paths.items()
]
# workflow template
container = convert_k8s_obj_to_json(processed_op.container)
step = {'name': processed_op.name}
step.update(container)
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
'steps': [step]
}
}
elif isinstance(op, dsl.ResourceOp):
# no output artifacts
output_artifacts = []
# task template
template = {
'apiVersion': tekton_api_version,
'kind': 'Task',
'metadata': {
'name': processed_op.name
},
'spec': {
"params": [
{
"description": "Action on the resource",
"name": "action",
"type": "string"
},
{
"default": "strategic",
"description":
"Merge strategy when using action patch",
"name": "merge-strategy",
"type": "string"
},
{
"description": "Content of the resource to deploy",
"name": "manifest",
"type": "string"
},
{
"default": "",
"description":
"An express to retrieval data from resource.",
"name": "output",
"type": "string"
},
{
"default": "",
"description":
"A label selector express to decide if the action on resource is success.",
"name": "success-condition",
"type": "string"
},
{
"default": "",
"description":
"A label selector express to decide if the action on resource is failure.",
"name": "failure-condition",
"type": "string"
},
{
"default":
"index.docker.io/fenglixa/kubeclient:v0.0.1", # Todo: The image need to be replaced, once there are official images from tekton
"description": "Kubectl wrapper image",
"name": "image",
"type": "string"
},
{
"default": "false",
"description":
"Enable set owner reference for created resource.",
"name": "set-ownerreference",
"type": "string"
}
],
'steps': [{
"args": [
"--action=$(params.action)",
"--merge-strategy=$(params.merge-strategy)",
"--manifest=$(params.manifest)",
"--output=$(params.output)",
"--success-condition=$(params.success-condition)",
"--failure-condition=$(params.failure-condition)",
"--set-ownerreference=$(params.set-ownerreference)"
],
"image":
"$(params.image)",
"name":
processed_op.name,
"resources": {}
}]
}
}
# initContainers
if processed_op.init_containers:
steps = processed_op.init_containers.copy()
steps.extend(template['spec']['steps'])
template['spec']['steps'] = steps
# inputs
input_artifact_paths = processed_op.input_artifact_paths if isinstance(
processed_op, dsl.ContainerOp) else None
artifact_arguments = processed_op.artifact_arguments if isinstance(
processed_op, dsl.ContainerOp) else None
inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths,
artifact_arguments)
if 'parameters' in inputs:
if isinstance(processed_op, dsl.ContainerOp):
template['spec']['params'] = inputs['parameters']
elif isinstance(op, dsl.ResourceOp):
template['spec']['params'].extend(inputs['parameters'])
elif 'artifacts' in inputs:
raise NotImplementedError("input artifacts are not yet implemented")
# outputs
if isinstance(op, dsl.ContainerOp):
op_outputs = processed_op.outputs
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
op_outputs = {}
param_outputs = {}
outputs_dict = _outputs_to_json(op, op_outputs, param_outputs,
output_artifacts)
if outputs_dict:
"""
Since Tekton results need to be under /tekton/results. If file output paths cannot be
configured to /tekton/results, we need to create the below copy step for moving
file outputs to the Tekton destination. BusyBox is recommended to be used on
small tasks because it's relatively lightweight and small compared to the ubuntu and
bash images.
- image: busybox
name: copy-results
script: |
#!/bin/sh
set -exo pipefail
cp $LOCALPATH $(results.data.path);
"""
template['spec']['results'] = []
copy_results_step = {
'image': 'busybox',
'name': 'copy-results',
'script': '#!/bin/sh\nset -exo pipefail\n'
}
volume_mount_step_template = []
volume_template = []
mounted_paths = []
for name, path in param_outputs.items():
name = name.replace(
'_', '-'
) # replace '_' to '-' since tekton results doesn't support underscore
template['spec']['results'].append({
'name': name,
'description': path
})
# replace all occurrences of the output file path with the Tekton output parameter expression
need_copy_step = True
for s in template['spec']['steps']:
if 'command' in s:
commands = []
for c in s['command']:
if path in c:
c = c.replace(path, '$(results.%s.path)' % name)
need_copy_step = False
commands.append(c)
s['command'] = commands
if 'args' in s:
args = []
for a in s['args']:
if path in a:
a = a.replace(path, '$(results.%s.path)' % name)
need_copy_step = False
args.append(a)
s['args'] = args
# If file output path cannot be found/replaced, use emptyDir to copy it to the tekton/results path
if need_copy_step:
copy_results_step['script'] = copy_results_step[
'script'] + 'cp ' + path + ' $(results.%s.path);' % name + '\n'
mountPath = path.rsplit("/", 1)[0]
if mountPath not in mounted_paths:
volume_mount_step_template.append({
'name':
name,
'mountPath':
path.rsplit("/", 1)[0]
})
volume_template.append({'name': name, 'emptyDir': {}})
mounted_paths.append(mountPath)
if mounted_paths:
copy_results_step['script'] = literal_str(
copy_results_step['script'])
template['spec']['steps'].append(copy_results_step)
template['spec']['stepTemplate'] = {}
template['spec']['stepTemplate'][
'volumeMounts'] = volume_mount_step_template
template['spec']['volumes'] = volume_template
# **********************************************************
# NOTE: the following features are still under development
# **********************************************************
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template.setdefault(
'metadata', {}) # Tekton change, don't wipe out existing metadata
if processed_op.pod_annotations:
template['metadata']['annotations'] = processed_op.pod_annotations
if processed_op.pod_labels:
template['metadata']['labels'] = processed_op.pod_labels
# sidecars
if processed_op.sidecars:
template['spec']['sidecars'] = processed_op.sidecars
# volumes
if processed_op.volumes:
template['spec']['volumes'] = template['spec'].get('volume', []) + [
convert_k8s_obj_to_json(volume) for volume in processed_op.volumes
]
template['spec']['volumes'].sort(key=lambda x: x['name'])
# Display name
if processed_op.display_name:
template.setdefault('metadata', {}).setdefault(
'annotations', {}
)['pipelines.kubeflow.org/task_display_name'] = processed_op.display_name
if isinstance(op, dsl.ContainerOp) and op._metadata:
import json
template.setdefault('metadata', {}).setdefault(
'annotations',
{})['pipelines.kubeflow.org/component_spec'] = json.dumps(
op._metadata.to_dict(), sort_keys=True)
return template