def encode(
self,
sub_expression_pb: placeholder_pb2.PlaceholderExpression,
component_spec: Optional[Type['types.ComponentSpec']] = None
) -> placeholder_pb2.PlaceholderExpression:
del component_spec # Unused by ConcatOperator
# ConcatOperator's proto version contains multiple placeholder expressions
# as operands. For convenience, the Python version is implemented taking
# only two operands.
if self._right:
# Resolve other expression
if isinstance(self._right, Placeholder):
other_expression = cast(Placeholder, self._right)
other_expression_pb = other_expression.encode()
else:
other_expression_pb = placeholder_pb2.PlaceholderExpression()
other_expression_pb.value.string_value = self._right
# Try combining with existing concat operator
if sub_expression_pb.HasField(
'operator') and sub_expression_pb.operator.HasField(
'concat_op'):
sub_expression_pb.operator.concat_op.expressions.append(
other_expression_pb)
return sub_expression_pb
else:
result = placeholder_pb2.PlaceholderExpression()
result.operator.concat_op.expressions.extend(
[sub_expression_pb, other_expression_pb])
return result
if self._left:
# Resolve other expression: left operand must be str
other_expression_pb = placeholder_pb2.PlaceholderExpression()
other_expression_pb.value.string_value = self._left
# Try combining with existing concat operator
if sub_expression_pb.HasField(
'operator') and sub_expression_pb.operator.HasField(
'concat_op'):
sub_expression_pb.operator.concat_op.expressions.insert(
0, other_expression_pb)
return sub_expression_pb
else:
result = placeholder_pb2.PlaceholderExpression()
result.operator.concat_op.expressions.extend(
[other_expression_pb, sub_expression_pb])
return result
raise RuntimeError(
'ConcatOperator does not have the other expression to concat.')
def resolve(self,
expression: placeholder_pb2.PlaceholderExpression) -> Any:
"""Recursively evaluates a placeholder expression."""
if expression.HasField("value"):
return getattr(expression.value,
expression.value.WhichOneof("value"))
elif expression.HasField("placeholder"):
return self._resolve_placeholder(expression.placeholder)
elif expression.HasField("operator"):
return self._resolve_placeholder_operator(expression.operator)
else:
raise ValueError("Unexpected placeholder expression type: "
f"{expression.WhichOneof('expression_type')}.")
def debug_str(expression: placeholder_pb2.PlaceholderExpression) -> str:
"""Gets the debug string of a placeholder expression proto.
Args:
expression: A placeholder expression proto.
Returns:
Debug string of the placeholder expression.
"""
if expression.HasField("value"):
value_field_name = expression.value.WhichOneof("value")
return f"\"{getattr(expression.value, value_field_name)}\""
if expression.HasField("placeholder"):
placeholder_pb = expression.placeholder
ph_names_map = {
placeholder_pb2.Placeholder.INPUT_ARTIFACT: "input",
placeholder_pb2.Placeholder.OUTPUT_ARTIFACT: "output",
placeholder_pb2.Placeholder.EXEC_PROPERTY: "exec_property",
placeholder_pb2.Placeholder.RUNTIME_INFO: "runtime_info",
placeholder_pb2.Placeholder.EXEC_INVOCATION: "execution_invocation"
}
ph_name = ph_names_map[placeholder_pb.type]
if placeholder_pb.key:
return f"{ph_name}(\"{placeholder_pb.key}\")"
else:
return f"{ph_name}()"
if expression.HasField("operator"):
operator_name = expression.operator.WhichOneof("operator_type")
operator_pb = getattr(expression.operator, operator_name)
if operator_name == "artifact_uri_op":
sub_expression_str = debug_str(operator_pb.expression)
if operator_pb.split:
return f"{sub_expression_str}.split_uri(\"{operator_pb.split}\")"
else:
return f"{sub_expression_str}.uri"
if operator_name == "artifact_value_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}.value"
if operator_name == "concat_op":
expression_str = " + ".join(
debug_str(e) for e in operator_pb.expressions)
return f"({expression_str})"
if operator_name == "index_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}[{operator_pb.index}]"
if operator_name == "proto_op":
sub_expression_str = debug_str(operator_pb.expression)
field_path = "".join(operator_pb.proto_field_path)
expression_str = f"{sub_expression_str}{field_path}"
if operator_pb.serialization_format:
format_str = placeholder_pb2.ProtoOperator.SerializationFormat.Name(
operator_pb.serialization_format)
return f"{expression_str}.serialize({format_str})"
return expression_str
if operator_name == "base64_encode_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}.b64encode()"
return "Unkown placeholder operator"
return "Unknown placeholder expression"
def placeholder_to_cel(
expression: placeholder_pb2.PlaceholderExpression) -> str:
"""Encodes a Predicate into a CEL string expression.
The CEL specification is at:
https://github.com/google/cel-spec/blob/master/doc/langdef.md
Args:
expression: A PlaceholderExpression proto descrbing a Predicate.
Returns:
A CEL expression in string format.
"""
if expression.HasField('value'):
value_field_name = expression.value.WhichOneof('value')
if value_field_name == 'int_value':
# In KFP IR, all values are defined as google.protobuf.Value,
# which does not differentiate between int and float. CEL's treats
# comparison between different types as an error. Hence we need to convert
# ints to floats for comparison in CEL.
return f'{float(expression.value.int_value)}'
if value_field_name == 'double_value':
return f'{expression.value.double_value}'
if value_field_name == 'string_value':
return f'\'{expression.value.string_value}\''
raise NotImplementedError(
'Only supports predicate with primitive type values.')
if expression.HasField('placeholder'):
placeholder_pb = expression.placeholder
# Predicates are always built from ChannelWrappedPlaceholder, which means
# a component can only write a predicate about its inputs. It doesn't make
# sense for a component to say "run only if my output is something."
if placeholder_pb.type != placeholder_pb2.Placeholder.INPUT_ARTIFACT:
raise NotImplementedError(
'Only supports accessing input artifact through placeholders on KFPv2.'
f'Got {placeholder_pb.type}.')
if not placeholder_pb.key:
raise ValueError(
'Only supports accessing placeholders with a key on KFPv2.')
# Note that because CEL automatically performs dynamic value conversion,
# we don't need type info for the oneof fields in google.protobuf.Value.
return f"inputs.artifacts['{placeholder_pb.key}'].artifacts"
if expression.HasField('operator'):
operator_name = expression.operator.WhichOneof('operator_type')
operator_pb = getattr(expression.operator, operator_name)
if operator_name == 'index_op':
sub_expression_str = placeholder_to_cel(operator_pb.expression)
return f'{sub_expression_str}[{operator_pb.index}]'
if operator_name == 'artifact_property_op':
sub_expression_str = placeholder_to_cel(operator_pb.expression)
# CEL's dynamic value conversion applies to here as well.
return f"{sub_expression_str}.metadata['{operator_pb.key}']"
if operator_name == 'artifact_uri_op':
sub_expression_str = placeholder_to_cel(operator_pb.expression)
if operator_pb.split:
raise NotImplementedError(
'Accessing artifact\'s split uri is unsupported.')
return f'{sub_expression_str}.uri'
if operator_name == 'concat_op':
expression_str = ' + '.join(
placeholder_to_cel(e) for e in operator_pb.expressions)
return f'({expression_str})'
if operator_name == 'compare_op':
lhs_str = placeholder_to_cel(operator_pb.lhs)
rhs_str = placeholder_to_cel(operator_pb.rhs)
if operator_pb.op == placeholder_pb2.ComparisonOperator.Operation.EQUAL:
op_str = '=='
elif operator_pb.op == placeholder_pb2.ComparisonOperator.Operation.LESS_THAN:
op_str = '<'
elif operator_pb.op == placeholder_pb2.ComparisonOperator.Operation.GREATER_THAN:
op_str = '>'
else:
return f'Unknown Comparison Operation {operator_pb.op}'
return f'({lhs_str} {op_str} {rhs_str})'
if operator_name == 'unary_logical_op':
expression_str = placeholder_to_cel(operator_pb.expression)
if operator_pb.op == placeholder_pb2.UnaryLogicalOperator.Operation.NOT:
op_str = '!'
else:
return f'Unknown Unary Logical Operation {operator_pb.op}'
return f'{op_str}({expression_str})'
if operator_name == 'binary_logical_op':
lhs_str = placeholder_to_cel(operator_pb.lhs)
rhs_str = placeholder_to_cel(operator_pb.rhs)
if operator_pb.op == placeholder_pb2.BinaryLogicalOperator.Operation.AND:
op_str = '&&'
elif operator_pb.op == placeholder_pb2.BinaryLogicalOperator.Operation.OR:
op_str = '||'
else:
return f'Unknown Binary Logical Operation {operator_pb.op}'
return f'({lhs_str} {op_str} {rhs_str})'
raise ValueError(f'Got unsupported placeholder operator {operator_name}.')
raise ValueError('Unknown placeholder expression.')
def debug_str(expression: placeholder_pb2.PlaceholderExpression) -> str:
"""Gets the debug string of a placeholder expression proto.
Args:
expression: A placeholder expression proto.
Returns:
Debug string of the placeholder expression.
"""
if expression.HasField("value"):
value_field_name = expression.value.WhichOneof("value")
return f"\"{getattr(expression.value, value_field_name)}\""
if expression.HasField("placeholder"):
placeholder_pb = expression.placeholder
ph_names_map = {
placeholder_pb2.Placeholder.INPUT_ARTIFACT: "input",
placeholder_pb2.Placeholder.OUTPUT_ARTIFACT: "output",
placeholder_pb2.Placeholder.EXEC_PROPERTY: "exec_property",
placeholder_pb2.Placeholder.RUNTIME_INFO: "runtime_info",
placeholder_pb2.Placeholder.EXEC_INVOCATION: "execution_invocation"
}
ph_name = ph_names_map[placeholder_pb.type]
if placeholder_pb.key:
return f"{ph_name}(\"{placeholder_pb.key}\")"
else:
return f"{ph_name}()"
if expression.HasField("operator"):
operator_name = expression.operator.WhichOneof("operator_type")
operator_pb = getattr(expression.operator, operator_name)
if operator_name == "artifact_uri_op":
sub_expression_str = debug_str(operator_pb.expression)
if operator_pb.split:
return f"{sub_expression_str}.split_uri(\"{operator_pb.split}\")"
else:
return f"{sub_expression_str}.uri"
if operator_name == "artifact_value_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}.value"
if operator_name == "artifact_property_op":
sub_expression_str = debug_str(operator_pb.expression)
if operator_pb.is_custom_property:
return f"{sub_expression_str}.custom_property(\"{operator_pb.key}\")"
else:
return f"{sub_expression_str}.property(\"{operator_pb.key}\")"
if operator_name == "concat_op":
expression_str = " + ".join(
debug_str(e) for e in operator_pb.expressions)
return f"({expression_str})"
if operator_name == "index_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}[{operator_pb.index}]"
if operator_name == "proto_op":
sub_expression_str = debug_str(operator_pb.expression)
field_path = "".join(operator_pb.proto_field_path)
expression_str = f"{sub_expression_str}{field_path}"
if operator_pb.serialization_format:
format_str = placeholder_pb2.ProtoOperator.SerializationFormat.Name(
operator_pb.serialization_format)
return f"{expression_str}.serialize({format_str})"
return expression_str
if operator_name == "base64_encode_op":
sub_expression_str = debug_str(operator_pb.expression)
return f"{sub_expression_str}.b64encode()"
if operator_name == "compare_op":
lhs_str = debug_str(operator_pb.lhs)
rhs_str = debug_str(operator_pb.rhs)
if operator_pb.op == _Operation.EQUAL.value:
op_str = "=="
elif operator_pb.op == _Operation.LESS_THAN.value:
op_str = "<"
elif operator_pb.op == _Operation.GREATER_THAN.value:
op_str = ">"
else:
return f"Unknown Comparison Operation {operator_pb.op}"
return f"({lhs_str} {op_str} {rhs_str})"
if operator_name == "unary_logical_op":
expression_str = debug_str(operator_pb.expression)
if operator_pb.op == _Operation.NOT.value:
op_str = "not"
else:
return f"Unknown Unary Logical Operation {operator_pb.op}"
return f"{op_str}({expression_str})"
if operator_name == "binary_logical_op":
lhs_str = debug_str(operator_pb.lhs)
rhs_str = debug_str(operator_pb.rhs)
if operator_pb.op == _Operation.AND.value:
op_str = "and"
elif operator_pb.op == _Operation.OR.value:
op_str = "or"
else:
return f"Unknown Binary Logical Operation {operator_pb.op}"
return f"({lhs_str} {op_str} {rhs_str})"
if operator_name == "list_serialization_op":
expression_str = debug_str(operator_pb.expression)
if operator_pb.serialization_format:
format_str = placeholder_pb2.ProtoOperator.SerializationFormat.Name(
operator_pb.serialization_format)
return f"{expression_str}.serialize_list({format_str})"
return expression_str
return "Unknown placeholder operator"
return "Unknown placeholder expression"