def _validate_kubernetes_secrets(
self, node_id: str, node_label: str, secrets: List[KubernetesSecret], response: ValidationResponse
) -> None:
"""
Checks the format of Kubernetes secrets to ensure they're in the correct form
e.g. FOO=SECRET_NAME:KEY
:param node_id: the unique ID of the node
:param node_label: the given node name or user customized name/label of the node
:param secrets: a KeyValueList of secrets to check
:param response: ValidationResponse containing the issue list to be updated
"""
for secret in secrets:
if not secret.name or not secret.key:
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidKubernetesSecret",
message=f"Environment variable '{secret.env_var}' has an improperly formatted representation of "
f"secret name and key.",
data={
"nodeID": node_id,
"nodeName": node_label,
"propertyName": KUBERNETES_SECRETS,
"value": KeyValueList.to_str(secret.env_var, f"{(secret.name or '')}:{(secret.key or '')}"),
},
)
continue
# Ensure the secret name is syntactically a valid Kubernetes resource name
if not is_valid_kubernetes_resource_name(secret.name):
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidKubernetesSecret",
message=f"Secret name '{secret.name}' is not a valid Kubernetes resource name.",
data={
"nodeID": node_id,
"nodeName": node_label,
"propertyName": KUBERNETES_SECRETS,
"value": KeyValueList.to_str(secret.env_var, f"{secret.name}:{secret.key}"),
},
)
# Ensure the secret key is a syntactically valid Kubernetes key
if not is_valid_kubernetes_key(secret.key):
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidKubernetesSecret",
message=f"Key '{secret.key}' is not a valid Kubernetes secret key.",
data={
"nodeID": node_id,
"nodeName": node_label,
"propertyName": KUBERNETES_SECRETS,
"value": KeyValueList.to_str(secret.env_var, f"{secret.name}:{secret.key}"),
},
)
def _validate_mounted_volumes(
self, node_id: str, node_label: str, volumes: List[VolumeMount], response: ValidationResponse
) -> None:
"""
Checks the format of mounted volumes to ensure they're in the correct form
e.g. foo/path=pvc_name
:param node_id: the unique ID of the node
:param node_label: the given node name or user customized name/label of the node
:param volumes: a KeyValueList of volumes to check
:param response: ValidationResponse containing the issue list to be updated
"""
for volume in volumes:
# Ensure the PVC name is syntactically a valid Kubernetes resource name
if not is_valid_kubernetes_resource_name(volume.pvc_name):
response.add_message(
severity=ValidationSeverity.Error,
message_type="invalidVolumeMount",
message=f"PVC name '{volume.pvc_name}' is not a valid Kubernetes resource name.",
data={
"nodeID": node_id,
"nodeName": node_label,
"propertyName": MOUNTED_VOLUMES,
"value": KeyValueList.to_str(volume.path, volume.pvc_name),
},
)
def test_envs_to_dict():
test_list = [
"TEST= one", "TEST_TWO=two ", "TEST_THREE =", " TEST_FOUR=1",
"TEST_FIVE = fi=ve "
]
test_dict_correct = {
"TEST": "one",
"TEST_TWO": "two",
"TEST_FOUR": "1",
"TEST_FIVE": "fi=ve"
}
assert KeyValueList(test_list).to_dict() == test_dict_correct
def remove_env_vars_with_matching_secrets(self):
"""
In the case of a matching key between env vars and kubernetes secrets,
prefer the Kubernetes Secret and remove the matching env var.
"""
env_vars = self.get_component_parameter(ENV_VARIABLES)
secrets = self.get_component_parameter(KUBERNETES_SECRETS)
if isinstance(env_vars, KeyValueList) and isinstance(
secrets, KeyValueList):
new_list = KeyValueList.difference(minuend=env_vars,
subtrahend=secrets)
self.set_component_parameter(ENV_VARIABLES, new_list)
def test_remove_env_vars_with_matching_secrets():
pipeline_json = _read_pipeline_resource(
"resources/sample_pipelines/pipeline_valid_with_pipeline_default.json")
pipeline_definition = PipelineDefinition(pipeline_definition=pipeline_json)
node = pipeline_definition.primary_pipeline.nodes.pop()
# Set kubernetes_secret property to have all the same keys as those in the env_vars property
kubernetes_secrets = KeyValueList(
["var1=name1:key1", "var2=name2:key2", "var3=name3:key3"])
node.set_component_parameter(KUBERNETES_SECRETS, kubernetes_secrets)
node.remove_env_vars_with_matching_secrets()
assert node.get_component_parameter(ENV_VARIABLES) == []
def convert_kv_properties(self, kv_properties: List[str]):
"""
Convert pipeline defaults-level list properties that have been identified
as sets of key-value pairs from a plain list type to the KeyValueList type.
"""
pipeline_defaults = self.get_property(PIPELINE_DEFAULTS, {})
for property_name, value in pipeline_defaults.items():
if property_name not in kv_properties:
continue
# Replace plain list with KeyValueList
pipeline_defaults[property_name] = KeyValueList(value)
if pipeline_defaults:
self.set_property(PIPELINE_DEFAULTS, pipeline_defaults)
def convert_kv_properties(self, kv_properties: List[str]):
"""
Convert node-level list properties that have been identified as sets of
key-value pairs from a plain list type to the KeyValueList type. If any
k-v property has already been converted to a KeyValueList, all k-v
properties are assumed to have already been converted.
"""
for kv_property in kv_properties:
value = self.get_component_parameter(kv_property)
if not value:
continue
if isinstance(value,
KeyValueList) or not isinstance(value[0], str):
# A KeyValueList instance implies all relevant properties have already been converted
# Similarly, if KeyValueList items aren't strings, this implies they have already been
# converted to the appropriate data class objects
return
# Convert plain list to KeyValueList
self.set_component_parameter(kv_property, KeyValueList(value))
def propagate_pipeline_default_properties(self):
"""
For any default pipeline properties set (e.g. runtime image, volume), propagate
the values to any nodes that do not set their own value for that property.
"""
# Convert any key-value list pipeline default properties to the KeyValueList type
kv_properties = PipelineDefinition.get_kv_properties()
self.primary_pipeline.convert_kv_properties(kv_properties)
pipeline_default_properties = self.primary_pipeline.get_property(
PIPELINE_DEFAULTS, {})
for node in self.pipeline_nodes:
if not Operation.is_generic_operation(node.op):
continue
# Convert any key-value list node properties to the KeyValueList type if not done already
node.convert_kv_properties(kv_properties)
for property_name, pipeline_default_value in pipeline_default_properties.items(
):
if not pipeline_default_value:
continue
node_value = node.get_component_parameter(property_name)
if not node_value:
node.set_component_parameter(property_name,
pipeline_default_value)
continue
if isinstance(pipeline_default_value,
KeyValueList) and isinstance(
node_value, KeyValueList):
merged_list = KeyValueList.merge(node_value,
pipeline_default_value)
node.set_component_parameter(property_name, merged_list)
if self.primary_pipeline.runtime_config != "local":
node.remove_env_vars_with_matching_secrets()
node.convert_data_class_properties()
def test_env_dict_to_list():
test_dict = {"TEST": "one", "TEST_TWO": "two", "TEST_FOUR": "1"}
test_list_correct = ["TEST=one", "TEST_TWO=two", "TEST_FOUR=1"]
assert KeyValueList.from_dict(test_dict) == test_list_correct