async def _validate_node_properties(
self,
pipeline_definition: PipelineDefinition,
pipeline_type: str,
pipeline_runtime: str,
response: ValidationResponse,
) -> None:
"""
Validates each of the node's structure for required fields/properties as well as
their values
:param pipeline_definition: the pipeline definition to be validated
:param pipeline_type: name of the pipeline runtime being used e.g. kfp, airflow, generic
:param pipeline_runtime: name of the pipeline runtime for execution e.g. kfp, airflow, local
:param response: ValidationResponse containing the issue list to be updated
"""
if pipeline_runtime:
# don't check if incompatible pipeline type and runtime
if not PipelineValidationManager._is_compatible_pipeline(pipeline_runtime, pipeline_type):
return
for pipeline in pipeline_definition.pipelines:
for node in pipeline.nodes:
if node.type == "execution_node":
if Operation.is_generic_operation(node.op):
self._validate_generic_node_properties(
node=node, response=response, pipeline_runtime=pipeline_runtime
)
# Validate runtime components against specific node properties in component registry
else:
await self._validate_custom_component_node_properties(
node=node,
response=response,
pipeline_runtime=pipeline_runtime,
pipeline_definition=pipeline_definition,
)
def propagate_pipeline_default_properties(self):
"""
For any default pipeline properties set (e.g. runtime image, volume), propagate
the values to any nodes that do not set their own value for that property.
"""
# Convert any key-value list pipeline default properties to the KeyValueList type
kv_properties = PipelineDefinition.get_kv_properties()
self.primary_pipeline.convert_kv_properties(kv_properties)
pipeline_default_properties = self.primary_pipeline.get_property(
PIPELINE_DEFAULTS, {})
for node in self.pipeline_nodes:
if not Operation.is_generic_operation(node.op):
continue
# Convert any key-value list node properties to the KeyValueList type if not done already
node.convert_kv_properties(kv_properties)
for property_name, pipeline_default_value in pipeline_default_properties.items(
):
if not pipeline_default_value:
continue
node_value = node.get_component_parameter(property_name)
if not node_value:
node.set_component_parameter(property_name,
pipeline_default_value)
continue
if isinstance(pipeline_default_value,
KeyValueList) and isinstance(
node_value, KeyValueList):
merged_list = KeyValueList.merge(node_value,
pipeline_default_value)
node.set_component_parameter(property_name, merged_list)
if self.primary_pipeline.runtime_config != "local":
node.remove_env_vars_with_matching_secrets()
node.convert_data_class_properties()