def test_convert_k8s_obj_to_json_accepts_dict(self): now = datetime.now() converted = convert_k8s_obj_to_json({ "ENV": "test", "number": 3, "list": [1, 2, 3], "time": now }) self.assertEqual(converted, { "ENV": "test", "number": 3, "list": [1, 2, 3], "time": now.isoformat() })
def _op_to_template(op: BaseOp, pipelinerun_output_artifacts={}, enable_artifacts=False): """Generate template given an operator inherited from BaseOp.""" # initial local variables for tracking volumes and artifacts volume_mount_step_template = [] volume_template = [] mounted_param_paths = [] replaced_param_list = [] artifact_to_result_mapping = {} # NOTE in-place update to BaseOp # replace all PipelineParams with template var strings processed_op = _process_base_ops(op) if isinstance(op, dsl.ContainerOp): # default output artifacts output_artifact_paths = OrderedDict(op.output_artifact_paths) # print(op.output_artifact_paths) # This should have been as easy as output_artifact_paths.update(op.file_outputs), # but the _outputs_to_json function changes the output names and we must do the same here, # so that the names are the same output_artifact_paths.update( sorted(((param.full_name, processed_op.file_outputs[param.name]) for param in processed_op.outputs.values()), key=lambda x: x[0])) output_artifacts = [ convert_k8s_obj_to_json( ArtifactLocation.create_artifact_for_s3( op.artifact_location, name=name, path=path, key='runs/$PIPELINERUN/$PIPELINETASK/' + name)) for name, path in output_artifact_paths.items() ] if enable_artifacts else [] # workflow template container = convert_k8s_obj_to_json(processed_op.container) # Calling containerOp step as "main" to align with Argo step = {'name': "main"} step.update(container) template = { 'apiVersion': tekton_api_version, 'kind': 'Task', 'metadata': { 'name': processed_op.name }, 'spec': { 'steps': [step] } } # Create output artifact tracking annotation. if enable_artifacts: for output_artifact in output_artifacts: output_annotation = pipelinerun_output_artifacts.get( processed_op.name, []) output_annotation.append({ 'name': output_artifact['name'], 'path': output_artifact['path'] }) pipelinerun_output_artifacts[ processed_op.name] = output_annotation elif isinstance(op, dsl.ResourceOp): # no output artifacts output_artifacts = [] # Flatten manifest because it needs to replace Argo variables manifest = yaml.dump(convert_k8s_obj_to_json( processed_op.k8s_resource), default_flow_style=False) argo_var = False if manifest.find('{{workflow.name}}') != -1: # Kubernetes Pod arguments only take $() as environment variables manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)") # Remove yaml quote in order to read bash variables manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest) argo_var = True # task template template = _get_resourceOp_template(op, processed_op.name, tekton_api_version, manifest, argo_var=argo_var) # initContainers if processed_op.init_containers: template['spec']['steps'] = _prepend_steps( processed_op.init_containers, template['spec']['steps']) # inputs input_artifact_paths = processed_op.input_artifact_paths if isinstance( processed_op, dsl.ContainerOp) else None artifact_arguments = processed_op.artifact_arguments if isinstance( processed_op, dsl.ContainerOp) else None inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths, artifact_arguments) if 'parameters' in inputs: if isinstance(processed_op, dsl.ContainerOp): template['spec']['params'] = inputs['parameters'] elif isinstance(op, dsl.ResourceOp): template['spec']['params'].extend(inputs['parameters']) if 'artifacts' in inputs: # Leave artifacts for big data passing template['spec']['artifacts'] = inputs['artifacts'] # outputs if isinstance(op, dsl.ContainerOp): op_outputs = processed_op.outputs param_outputs = processed_op.file_outputs elif isinstance(op, dsl.ResourceOp): op_outputs = {} param_outputs = {} outputs_dict = _outputs_to_json(op, op_outputs, param_outputs, output_artifacts) if outputs_dict: copy_results_step = _process_parameters( processed_op, template, outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping, mounted_param_paths) copy_artifacts_step = _process_output_artifacts( outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping) if mounted_param_paths: template['spec']['steps'].append(copy_results_step) _update_volumes(template, volume_mount_step_template, volume_template) if copy_artifacts_step: template['spec']['steps'].append(copy_artifacts_step) # metadata if processed_op.pod_annotations or processed_op.pod_labels: template.setdefault( 'metadata', {}) # Tekton change, don't wipe out existing metadata if processed_op.pod_annotations: template['metadata']['annotations'] = { sanitize_k8s_name(key, allow_capital_underscore=True, allow_dot=True, allow_slash=True, max_length=253): value for key, value in processed_op.pod_annotations.items() } if processed_op.pod_labels: template['metadata']['labels'] = { sanitize_k8s_name(key, allow_capital_underscore=True, allow_dot=True, allow_slash=True, max_length=253): sanitize_k8s_name(value, allow_capital_underscore=True, allow_dot=True) for key, value in processed_op.pod_labels.items() } # sidecars if processed_op.sidecars: template['spec']['sidecars'] = processed_op.sidecars # volumes if processed_op.volumes: template['spec']['volumes'] = template['spec'].get('volume', []) + [ convert_k8s_obj_to_json(volume) for volume in processed_op.volumes ] template['spec']['volumes'].sort(key=lambda x: x['name']) # Display name if processed_op.display_name: template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/task_display_name'] = \ processed_op.display_name if isinstance(op, dsl.ContainerOp) and op._metadata: template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/component_spec'] = \ json.dumps(op._metadata.to_dict(), sort_keys=True) if isinstance(op, dsl.ContainerOp) and op.execution_options: if op.execution_options.caching_strategy.max_cache_staleness: template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/max_cache_staleness'] = \ str(op.execution_options.caching_strategy.max_cache_staleness) return template
def _op_to_template(op: BaseOp, pipelinerun_output_artifacts={}, artifact_items={}): """Generate template given an operator inherited from BaseOp.""" # Display name if op.display_name: op.add_pod_annotation('pipelines.kubeflow.org/task_display_name', op.display_name) # initial local variables for tracking volumes and artifacts volume_mount_step_template = [] volume_template = [] mounted_param_paths = [] replaced_param_list = [] artifact_to_result_mapping = {} # NOTE in-place update to BaseOp # replace all PipelineParams with template var strings processed_op = _process_base_ops(op) if isinstance(op, dsl.ContainerOp): # default output artifacts output_artifact_paths = OrderedDict(op.output_artifact_paths) # print(op.output_artifact_paths) # This should have been as easy as output_artifact_paths.update(op.file_outputs), # but the _outputs_to_json function changes the output names and we must do the same here, # so that the names are the same output_artifact_paths.update( sorted(((param.full_name, processed_op.file_outputs[param.name]) for param in processed_op.outputs.values()), key=lambda x: x[0])) output_artifacts = [{ 'name': name, 'path': path } for name, path in output_artifact_paths.items()] # workflow template container = convert_k8s_obj_to_json(processed_op.container) # Calling containerOp step as "main" to align with Argo step = {'name': "main"} step.update(container) template = { 'apiVersion': tekton_api_version, 'kind': 'Task', 'metadata': { 'name': processed_op.name }, 'spec': { 'steps': [step] } } # Create output artifact tracking annotation. for output_artifact in output_artifacts: output_annotation = pipelinerun_output_artifacts.get( processed_op.name, []) output_annotation.append({ 'name': output_artifact.get('name', ''), 'path': output_artifact.get('path', ''), 'key': "artifacts/$PIPELINERUN/%s/%s.tgz" % (processed_op.name, output_artifact.get('name', '').replace( processed_op.name + '-', '')) }) pipelinerun_output_artifacts[processed_op.name] = output_annotation elif isinstance(op, dsl.ResourceOp): # no output artifacts output_artifacts = [] # Flatten manifest because it needs to replace Argo variables manifest = yaml.dump(convert_k8s_obj_to_json( processed_op.k8s_resource), default_flow_style=False) argo_var = False if manifest.find('{{workflow.name}}') != -1: # Kubernetes Pod arguments only take $() as environment variables manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)") # Remove yaml quote in order to read bash variables manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest) argo_var = True # task template template = _get_resourceOp_template(op, processed_op.name, tekton_api_version, manifest, argo_var=argo_var) # initContainers if processed_op.init_containers: template['spec']['steps'] = _prepend_steps( processed_op.init_containers, template['spec']['steps']) # inputs input_artifact_paths = processed_op.input_artifact_paths if isinstance( processed_op, dsl.ContainerOp) else None artifact_arguments = processed_op.artifact_arguments if isinstance( processed_op, dsl.ContainerOp) else None inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths, artifact_arguments) if 'parameters' in inputs: if isinstance(processed_op, dsl.ContainerOp): template['spec']['params'] = inputs['parameters'] elif isinstance(op, dsl.ResourceOp): template['spec']['params'].extend(inputs['parameters']) if 'artifacts' in inputs: # Leave artifacts for big data passing template['spec']['artifacts'] = inputs['artifacts'] # outputs if isinstance(op, dsl.ContainerOp): op_outputs = processed_op.outputs param_outputs = processed_op.file_outputs elif isinstance(op, dsl.ResourceOp): op_outputs = {} param_outputs = {} outputs_dict = _outputs_to_json(op, op_outputs, param_outputs, output_artifacts) artifact_items[op.name] = artifact_items.get(op.name, []) if outputs_dict: copy_results_step = _process_parameters( processed_op, template, outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping, mounted_param_paths) _process_output_artifacts(outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping, artifact_items[op.name]) if mounted_param_paths: template['spec']['steps'].append(copy_results_step) _update_volumes(template, volume_mount_step_template, volume_template) # metadata if processed_op.pod_annotations or processed_op.pod_labels: template.setdefault( 'metadata', {}) # Tekton change, don't wipe out existing metadata if processed_op.pod_annotations: template['metadata']['annotations'] = { sanitize_k8s_name(key, allow_capital_underscore=True, allow_dot=True, allow_slash=True, max_length=253): value for key, value in processed_op.pod_annotations.items() } if processed_op.pod_labels: template['metadata']['labels'] = { sanitize_k8s_name(key, allow_capital_underscore=True, allow_dot=True, allow_slash=True, max_length=253): sanitize_k8s_name(value, allow_capital_underscore=True, allow_dot=True) for key, value in processed_op.pod_labels.items() } # sidecars if processed_op.sidecars: template['spec']['sidecars'] = processed_op.sidecars # volumes if processed_op.volumes: template['spec']['volumes'] = template['spec'].get('volumes', []) + [ convert_k8s_obj_to_json(volume) for volume in processed_op.volumes ] template['spec']['volumes'].sort(key=lambda x: x['name']) if isinstance(op, dsl.ContainerOp ) and op._metadata and GENERATE_COMPONENT_SPEC_ANNOTATIONS: component_spec_dict = op._metadata.to_dict() component_spec_digest = hashlib.sha256( json.dumps(component_spec_dict, sort_keys=True).encode()).hexdigest() component_name = component_spec_dict.get('name', op.name) component_version = component_name + '@sha256=' + component_spec_digest digested_component_spec_dict = { 'name': component_name, 'outputs': component_spec_dict.get('outputs', []), 'version': component_version } template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/component_spec_digest'] = \ json.dumps(digested_component_spec_dict, sort_keys=True) if isinstance(op, dsl.ContainerOp) and op.execution_options: if op.execution_options.caching_strategy.max_cache_staleness: template.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/max_cache_staleness'] = \ str(op.execution_options.caching_strategy.max_cache_staleness) # Sort and arrange results based on provided estimate size and process results in multi-steps if the result sizes are too big. result_size_map = "{}" if processed_op.pod_annotations: result_size_map = processed_op.pod_annotations.get( "tekton-result-sizes", "{}") # Only sort and arrange results when the estimated sizes are given. if result_size_map and result_size_map != "{}": try: result_size_map = json.loads(result_size_map) except ValueError: raise ("tekton-result-sizes annotation is not a valid JSON") # Normalize estimated result size keys. result_size_map = { sanitize_k8s_name(key, allow_capital_underscore=True): value for key, value in result_size_map.items() } # Sort key orders based on values result_size_map = dict( sorted(result_size_map.items(), key=lambda item: item[1], reverse=True)) max_byte_size = 2048 verified_result_size_map = {0: {}} op_result_names = [ name['name'] for name in template['spec']['results'] ] step_bins = {0: 0} step_counter = 0 # Group result files to not exceed max_byte_size as a bin packing problem # Results are sorted from large to small, each value will loop over each bin to determine can it fit in the existing bins. for key, value in result_size_map.items(): try: value = int(value) except ValueError: raise ( "Estimated value for result %s is %s, but it needs to be an integer." % (key, value)) if key in op_result_names: packed_index = -1 # Look for bin that can fit the result value for i in range(len(step_bins)): if step_bins[i] + value > max_byte_size: continue step_bins[i] = step_bins[i] + value packed_index = i break # If no bin can fit the value, create a new bin to store the value if packed_index < 0: step_counter += 1 if value > max_byte_size: logging.warning( "The estimated size for parameter %s is %sB which is more than 2KB, " "consider passing this value as artifact instead of output parameter." % (key, str(value))) step_bins[step_counter] = value verified_result_size_map[step_counter] = {} packed_index = step_counter verified_result_size_map[packed_index][key] = value else: logging.warning( "The esitmated size for parameter %s does not exist in the task %s." "Please correct the task annotations with the correct parameter key" % (key, op.name)) missing_param_estimation = [] for result_name in op_result_names: if result_name not in result_size_map.keys(): missing_param_estimation.append(result_name) if missing_param_estimation: logging.warning( "The following output parameter estimations are missing in task %s: Missing params: %s." % (op.name, missing_param_estimation)) # Move results between the Tekton home and result directories if there are more than one step if step_counter > 0: for step in template['spec']['steps']: if step['name'] == 'main': for key in result_size_map.keys(): # Replace main step results that are not in the first bin to the Tekton home path if key not in verified_result_size_map[0].keys(): sanitize_key = sanitize_k8s_name(key) for i, a in enumerate(step['args']): a = a.replace( '$(results.%s.path)' % sanitize_key, '%s%s' % (TEKTON_HOME_RESULT_PATH, sanitize_key)) step['args'][i] = a for i, c in enumerate(step['command']): c = c.replace( '$(results.%s.path)' % sanitize_key, '%s%s' % (TEKTON_HOME_RESULT_PATH, sanitize_key)) step['command'][i] = c # Append new steps to move result files between each step, so Tekton controller can record all results without # exceeding the Kubernetes termination log limit. for i in range(1, step_counter + 1): copy_result_step = _get_copy_result_step_template( i, verified_result_size_map) template['spec']['steps'].append(copy_result_step) # Update actifact item location to the latest stage in order to properly track and store all the artifacts. for i, artifact in enumerate(artifact_items[op.name]): if artifact[0] not in verified_result_size_map[step_counter].keys( ): artifact[1] = '%s%s' % (TEKTON_HOME_RESULT_PATH, sanitize_k8s_name(artifact[0])) artifact_items[op.name][i] = artifact return template
def _workflow_with_pipelinerun(self, task_refs, pipeline, pipeline_template, workflow): """ Generate pipelinerun template """ pipelinerun = { 'apiVersion': tekton_api_version, 'kind': 'PipelineRun', 'metadata': { 'name': pipeline_template['metadata']['name'] + '-run' }, 'spec': { 'params': [{ 'name': p['name'], 'value': p.get('default', '') } for p in pipeline_template['spec']['params']], 'pipelineRef': { 'name': pipeline_template['metadata']['name'] } } } # Generate TaskRunSpec PodTemplate:s task_run_spec = [] for task in task_refs: op = pipeline.ops.get(task['name']) task_spec = { "pipelineTaskName": task['name'], "taskPodTemplate": {} } if op.affinity: task_spec["taskPodTemplate"][ "affinity"] = convert_k8s_obj_to_json(op.affinity) if op.tolerations: task_spec["taskPodTemplate"]['tolerations'] = op.tolerations if op.node_selector: task_spec["taskPodTemplate"]['nodeSelector'] = op.node_selector if bool(task_spec["taskPodTemplate"]): task_run_spec.append(task_spec) if len(task_run_spec) > 0: pipelinerun['spec']['taskRunSpec'] = task_run_spec # add workflow level timeout to pipeline run if pipeline.conf.timeout: pipelinerun['spec']['timeout'] = '%ds' % pipeline.conf.timeout # generate the Tekton service account template for image pull secret service_template = {} if len(pipeline.conf.image_pull_secrets) > 0: service_template = { 'apiVersion': 'v1', 'kind': 'ServiceAccount', 'metadata': { 'name': pipelinerun['metadata']['name'] + '-sa' } } for image_pull_secret in pipeline.conf.image_pull_secrets: service_template['imagePullSecrets'] = [{ 'name': image_pull_secret.name }] if service_template: workflow = workflow + [service_template] pipelinerun['spec']['serviceAccountName'] = service_template[ 'metadata']['name'] workflow = workflow + [pipelinerun] return workflow
def _create_pipeline_workflow(self, args, pipeline, op_transformers=None, pipeline_conf=None) \ -> Dict[Text, Any]: """Create workflow for the pipeline.""" # Input Parameters params = [] for arg in args: param = {'name': arg.name} if arg.value is not None: if isinstance(arg.value, (list, tuple, dict)): param['default'] = json.dumps(arg.value, sort_keys=True) else: param['default'] = str(arg.value) params.append(param) # TODO: task templates? # generate Tekton tasks from pipeline ops raw_templates = self._create_dag_templates(pipeline, op_transformers, params) # generate task and condition reference list for the Tekton Pipeline condition_refs = {} # TODO task_refs = [] templates = [] condition_task_refs = {} for template in raw_templates: # TODO Allow an opt-out for the condition_template if template['kind'] == 'Condition': condition_task_ref = [{ 'name': template['metadata']['name'], 'params': [{ 'name': p['name'], 'value': p.get('value', '') } for p in template['spec'].get('params', [])], 'taskSpec': _get_super_condition_template(), }] condition_refs[template['metadata']['name']] = [{ 'input': '$(tasks.%s.results.status)' % template['metadata']['name'], 'operator': 'in', 'values': ['true'] }] condition_task_refs[template['metadata'] ['name']] = condition_task_ref else: templates.append(template) task_ref = { 'name': template['metadata']['name'], 'params': [{ 'name': p['name'], 'value': p.get('default', '') } for p in template['spec'].get('params', [])], 'taskSpec': template['spec'], } if template['metadata'].get('labels', None): task_ref['taskSpec']['metadata'] = task_ref[ 'taskSpec'].get('metadata', {}) task_ref['taskSpec']['metadata']['labels'] = template[ 'metadata']['labels'] if template['metadata'].get('annotations', None): task_ref['taskSpec']['metadata'] = task_ref[ 'taskSpec'].get('metadata', {}) task_ref['taskSpec']['metadata']['annotations'] = template[ 'metadata']['annotations'] task_refs.append(task_ref) # process input parameters from upstream tasks for conditions and pair conditions with their ancestor conditions opsgroup_stack = [pipeline.groups[0]] condition_stack = [None] while opsgroup_stack: cur_opsgroup = opsgroup_stack.pop() most_recent_condition = condition_stack.pop() if cur_opsgroup.type == 'condition': condition_task_ref = condition_task_refs[cur_opsgroup.name][0] condition = cur_opsgroup.condition input_params = [] # Process input parameters if needed if isinstance(condition.operand1, dsl.PipelineParam): if condition.operand1.op_name: operand_value = '$(tasks.' + condition.operand1.op_name + '.results.' + sanitize_k8s_name( condition.operand1.name) + ')' else: operand_value = '$(params.' + condition.operand1.name + ')' input_params.append(operand_value) if isinstance(condition.operand2, dsl.PipelineParam): if condition.operand2.op_name: operand_value = '$(tasks.' + condition.operand2.op_name + '.results.' + sanitize_k8s_name( condition.operand2.name) + ')' else: operand_value = '$(params.' + condition.operand2.name + ')' input_params.append(operand_value) for param_iter in range(len(input_params)): condition_task_ref['params'][param_iter][ 'value'] = input_params[param_iter] opsgroup_stack.extend(cur_opsgroup.groups) condition_stack.extend([ most_recent_condition for x in range(len(cur_opsgroup.groups)) ]) # add task dependencies and add condition refs to the task ref that depends on the condition op_name_to_parent_groups = self._get_groups_for_ops(pipeline.groups[0]) for task in task_refs: op = pipeline.ops.get(task['name']) parent_group = op_name_to_parent_groups.get(task['name'], []) if parent_group: if condition_refs.get(parent_group[-2], []): task['when'] = condition_refs.get( op_name_to_parent_groups[task['name']][-2], []) if op.dependent_names: task['runAfter'] = op.dependent_names # process input parameters from upstream tasks pipeline_param_names = [p['name'] for p in params] for task in task_refs: op = pipeline.ops.get(task['name']) for tp in task.get('params', []): if tp['name'] in pipeline_param_names: tp['value'] = '$(params.%s)' % tp['name'] else: for pp in op.inputs: if tp['name'] == pp.full_name: tp['value'] = '$(tasks.%s.results.%s)' % ( pp.op_name, pp.name) # Create input artifact tracking annotation input_annotation = self.input_artifacts.get( task['name'], []) input_annotation.append({ 'name': tp['name'], 'parent_task': pp.op_name }) self.input_artifacts[ task['name']] = input_annotation break # add retries params for task in task_refs: op = pipeline.ops.get(task['name']) if op.num_retries: task['retries'] = op.num_retries # add timeout params to task_refs, instead of task. for task in task_refs: op = pipeline.ops.get(task['name']) if not TEKTON_GLOBAL_DEFAULT_TIMEOUT or op.timeout: task['timeout'] = '%ds' % op.timeout # handle resourceOp cases in pipeline self._process_resourceOp(task_refs, pipeline) # handle exit handler in pipeline finally_tasks = [] for task in task_refs: op = pipeline.ops.get(task['name']) if op.is_exit_handler: finally_tasks.append(task) task_refs = [ task for task in task_refs if not pipeline.ops.get(task['name']).is_exit_handler ] # process loop parameters, keep this section in the behind of other processes, ahead of gen pipeline root_group = pipeline.groups[0] op_name_to_for_loop_op = self._get_for_loop_ops(root_group) if op_name_to_for_loop_op: for loop_param in op_name_to_for_loop_op.values(): if loop_param.items_is_pipeline_param is True: raise NotImplementedError( "dynamic params are not yet implemented") include_loop_task_refs = [] for task in task_refs: with_loop_task = self._get_loop_task(task, op_name_to_for_loop_op) include_loop_task_refs.extend(with_loop_task) task_refs = include_loop_task_refs # Flatten condition task condition_task_refs_temp = [] for condition_task_ref in condition_task_refs.values(): for ref in condition_task_ref: condition_task_refs_temp.append(ref) condition_task_refs = condition_task_refs_temp # TODO: generate the PipelineRun template pipeline_run = { 'apiVersion': tekton_api_version, 'kind': 'PipelineRun', 'metadata': { 'name': sanitize_k8s_name(pipeline.name or 'Pipeline', suffix_space=4), # 'labels': get_default_telemetry_labels(), 'annotations': { 'tekton.dev/output_artifacts': json.dumps(self.output_artifacts, sort_keys=True), 'tekton.dev/input_artifacts': json.dumps(self.input_artifacts, sort_keys=True), 'tekton.dev/artifact_bucket': DEFAULT_ARTIFACT_BUCKET, 'tekton.dev/artifact_endpoint': DEFAULT_ARTIFACT_ENDPOINT, 'tekton.dev/artifact_endpoint_scheme': DEFAULT_ARTIFACT_ENDPOINT_SCHEME, 'tekton.dev/artifact_items': json.dumps(self.artifact_items, sort_keys=True), 'sidecar.istio.io/inject': 'false' # disable Istio inject since Tekton cannot run with Istio sidecar } }, 'spec': { 'params': [{ 'name': p['name'], 'value': p.get('default', '') } for p in params], 'pipelineSpec': { 'params': params, 'tasks': task_refs + condition_task_refs, 'finally': finally_tasks } } } # TODO: pipelineRun additions # Generate TaskRunSpec PodTemplate:s task_run_spec = [] for task in task_refs: # TODO: should loop-item tasks be included here? if LoopArguments.LOOP_ITEM_NAME_BASE in task['name']: task_name = re.sub( r'-%s-.+$' % LoopArguments.LOOP_ITEM_NAME_BASE, '', task['name']) else: task_name = task['name'] op = pipeline.ops.get(task_name) if not op: raise RuntimeError("unable to find op with name '%s'" % task["name"]) task_spec = { "pipelineTaskName": task['name'], "taskPodTemplate": {} } if op.affinity: task_spec["taskPodTemplate"][ "affinity"] = convert_k8s_obj_to_json(op.affinity) if op.tolerations: task_spec["taskPodTemplate"]['tolerations'] = op.tolerations if op.node_selector: task_spec["taskPodTemplate"]['nodeSelector'] = op.node_selector if bool(task_spec["taskPodTemplate"]): task_run_spec.append(task_spec) if len(task_run_spec) > 0: pipeline_run['spec']['taskRunSpecs'] = task_run_spec # add workflow level timeout to pipeline run if not TEKTON_GLOBAL_DEFAULT_TIMEOUT or pipeline.conf.timeout: pipeline_run['spec']['timeout'] = '%ds' % pipeline.conf.timeout # generate the Tekton podTemplate for image pull secret if len(pipeline.conf.image_pull_secrets) > 0: pipeline_run['spec']['podTemplate'] = pipeline_run['spec'].get( 'podTemplate', {}) pipeline_run['spec']['podTemplate']['imagePullSecrets'] = [{ "name": s.name } for s in pipeline.conf.image_pull_secrets] workflow = pipeline_run return workflow
def _workflow_with_pipelinerun(self, task_refs, pipeline, pipeline_template, workflow): """ Generate pipelinerun template """ pipelinerun = { 'apiVersion': tekton_api_version, 'kind': 'PipelineRun', 'metadata': { 'name': sanitize_k8s_name(pipeline_template['metadata']['name'], suffix_space=4) + '-run', 'annotation': { 'tekton.dev/output_artifacts': json.dumps(self.output_artifacts), 'tekton.dev/input_artifacts': json.dumps(self.input_artifacts) } }, 'spec': { 'params': [{ 'name': p['name'], 'value': p.get('default', '') } for p in pipeline_template['spec']['params']], 'pipelineRef': { 'name': pipeline_template['metadata']['name'] } } } # Generate TaskRunSpecs PodTemplate:s task_run_spec = [] for task in task_refs: op = pipeline.ops.get(task['name']) task_spec = { "pipelineTaskName": task['name'], "taskPodTemplate": {} } if op.affinity: task_spec["taskPodTemplate"][ "affinity"] = convert_k8s_obj_to_json(op.affinity) if op.tolerations: task_spec["taskPodTemplate"]['tolerations'] = op.tolerations if op.node_selector: task_spec["taskPodTemplate"]['nodeSelector'] = op.node_selector if bool(task_spec["taskPodTemplate"]): task_run_spec.append(task_spec) if len(task_run_spec) > 0: pipelinerun['spec']['taskRunSpecs'] = task_run_spec # add workflow level timeout to pipeline run if pipeline.conf.timeout: pipelinerun['spec']['timeout'] = '%ds' % pipeline.conf.timeout # generate the Tekton podTemplate for image pull secret if len(pipeline.conf.image_pull_secrets) > 0: pipelinerun['spec']['podTemplate'] = pipelinerun['spec'].get( 'podTemplate', {}) pipelinerun['spec']['podTemplate']['imagePullSecrets'] = [{ "name": s.name } for s in pipeline.conf.image_pull_secrets] workflow = workflow + [pipelinerun] return workflow
def _op_to_template(op: BaseOp, enable_artifacts=False): """Generate template given an operator inherited from BaseOp.""" # initial local variables for tracking volumes and artifacts volume_mount_step_template = [] volume_template = [] mounted_param_paths = [] replaced_param_list = [] artifact_to_result_mapping = {} # NOTE in-place update to BaseOp # replace all PipelineParams with template var strings processed_op = _process_base_ops(op) if isinstance(op, dsl.ContainerOp): # default output artifacts output_artifact_paths = OrderedDict(op.output_artifact_paths) # print(op.output_artifact_paths) # This should have been as easy as output_artifact_paths.update(op.file_outputs), # but the _outputs_to_json function changes the output names and we must do the same here, # so that the names are the same output_artifact_paths.update( sorted(((param.full_name, processed_op.file_outputs[param.name]) for param in processed_op.outputs.values()), key=lambda x: x[0])) output_artifacts = [ convert_k8s_obj_to_json( ArtifactLocation.create_artifact_for_s3( op.artifact_location, name=name, path=path, key='runs/$PIPELINERUN/$PODNAME/' + name)) for name, path in output_artifact_paths.items() ] if enable_artifacts else [] # workflow template container = convert_k8s_obj_to_json(processed_op.container) step = {'name': processed_op.name} step.update(container) template = { 'apiVersion': tekton_api_version, 'kind': 'Task', 'metadata': { 'name': processed_op.name }, 'spec': { 'steps': [step] } } elif isinstance(op, dsl.ResourceOp): # no output artifacts output_artifacts = [] # Flatten manifest because it needs to replace Argo variables manifest = yaml.dump(convert_k8s_obj_to_json( processed_op.k8s_resource), default_flow_style=False) argo_var = False if manifest.find('{{workflow.name}}') != -1: # Kubernetes Pod arguments only take $() as environment variables manifest = manifest.replace('{{workflow.name}}', "$(PIPELINERUN)") # Remove yaml quote in order to read bash variables manifest = re.sub('name: \'([^\']+)\'', 'name: \g<1>', manifest) argo_var = True # task template template = _get_resourceOp_template(op, processed_op.name, tekton_api_version, manifest, argo_var=argo_var) # initContainers if processed_op.init_containers: template['spec']['steps'] = _prepend_steps( processed_op.init_containers, template['spec']['steps']) # inputs input_artifact_paths = processed_op.input_artifact_paths if isinstance( processed_op, dsl.ContainerOp) else None artifact_arguments = processed_op.artifact_arguments if isinstance( processed_op, dsl.ContainerOp) else None inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths, artifact_arguments) if 'parameters' in inputs: if isinstance(processed_op, dsl.ContainerOp): template['spec']['params'] = inputs['parameters'] elif isinstance(op, dsl.ResourceOp): template['spec']['params'].extend(inputs['parameters']) if 'artifacts' in inputs: # The input artifacts in KFP is not pulling from s3, it will always be passed as a raw input. # Visit https://github.com/kubeflow/pipelines/issues/336 for more details on the implementation. copy_inputs_step = _get_base_step('copy-inputs') for artifact in inputs['artifacts']: if 'raw' in artifact: copy_inputs_step['script'] += 'echo -n "%s" > %s\n' % ( artifact['raw']['data'], artifact['path']) mount_path = artifact['path'].rsplit("/", 1)[0] if mount_path not in mounted_param_paths: _add_mount_path(artifact['name'], artifact['path'], mount_path, volume_mount_step_template, volume_template, mounted_param_paths) template['spec']['steps'] = _prepend_steps([copy_inputs_step], template['spec']['steps']) _update_volumes(template, volume_mount_step_template, volume_template) # outputs if isinstance(op, dsl.ContainerOp): op_outputs = processed_op.outputs param_outputs = processed_op.file_outputs elif isinstance(op, dsl.ResourceOp): op_outputs = {} param_outputs = {} outputs_dict = _outputs_to_json(op, op_outputs, param_outputs, output_artifacts) if outputs_dict: copy_results_step = _process_parameters( processed_op, template, outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping, mounted_param_paths) copy_artifacts_step = _process_output_artifacts( outputs_dict, volume_mount_step_template, volume_template, replaced_param_list, artifact_to_result_mapping) if mounted_param_paths: template['spec']['steps'].append(copy_results_step) _update_volumes(template, volume_mount_step_template, volume_template) if copy_artifacts_step: template['spec']['steps'].append(copy_artifacts_step) # metadata if processed_op.pod_annotations or processed_op.pod_labels: template.setdefault( 'metadata', {}) # Tekton change, don't wipe out existing metadata if processed_op.pod_annotations: template['metadata']['annotations'] = processed_op.pod_annotations if processed_op.pod_labels: template['metadata']['labels'] = processed_op.pod_labels # sidecars if processed_op.sidecars: template['spec']['sidecars'] = processed_op.sidecars # volumes if processed_op.volumes: template['spec']['volumes'] = template['spec'].get('volume', []) + [ convert_k8s_obj_to_json(volume) for volume in processed_op.volumes ] template['spec']['volumes'].sort(key=lambda x: x['name']) # Display name if processed_op.display_name: template.setdefault('metadata', {}).setdefault( 'annotations', {} )['pipelines.kubeflow.org/task_display_name'] = processed_op.display_name if isinstance(op, dsl.ContainerOp) and op._metadata: template.setdefault('metadata', {}).setdefault( 'annotations', {})['pipelines.kubeflow.org/component_spec'] = json.dumps( op._metadata.to_dict(), sort_keys=True) if isinstance(op, dsl.ContainerOp) and op.execution_options: if op.execution_options.caching_strategy.max_cache_staleness: template.setdefault('metadata', {}).setdefault( 'annotations', {})['pipelines.kubeflow.org/max_cache_staleness'] = str( op.execution_options.caching_strategy.max_cache_staleness) return template