def testExecPropertyProtoField(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('proto')[0].a.b['c'], """
operator {
proto_op {
expression {
operator {
index_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "proto"
}
}
index: 0
}
}
}
proto_field_path: ".a"
proto_field_path: ".b"
proto_field_path: "['c']"
}
}
""")
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('proto').a['b'].c[1], """
operator {
index_op {
expression {
operator {
proto_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "proto"
}
}
proto_field_path: ".a"
proto_field_path: "['b']"
proto_field_path: ".c"
}
}
}
index: 1
}
}
""")
def _recursively_encode(
self,
ph: Union[placeholders.CommandlineArgumentType,
placeholder.Placeholder, str],
component_spec: Optional[types.ComponentSpec] = None
) -> Union[str, placeholder.Placeholder]:
"""This method recursively encodes placeholders.CommandlineArgumentType.
The recursion ending condision is that the input ph is alerady a string
or placeholder.Placeholder.
Args:
ph: The placeholder to encode.
component_spec: Optional. The ComponentSpec to help with the encoding.
Returns:
The encoded placeholder in the type of string or placeholder.Placeholder.
"""
if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder):
# If there is no place holder. Or if the placeholder is already a
# new style placeholder.
# No further encoding is needed.
return cast(Union[str, placeholder.Placeholder], ph)
elif isinstance(ph, placeholders.InputValuePlaceholder):
if not component_spec:
raise ValueError(
'Requires component spec to encode InputValuePlaceholder.')
if ph.input_name in component_spec.INPUTS:
return placeholder.input(ph.input_name)[0].value
elif ph.input_name in component_spec.PARAMETERS:
return placeholder.exec_property(ph.input_name)
else:
raise ValueError(
'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.'
)
elif isinstance(ph, placeholders.InputUriPlaceholder):
if component_spec and ph.input_name not in component_spec.INPUTS:
raise ValueError(
'For InputUriPlaceholder, input name must be in component\'s INPUTS.'
)
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
if component_spec and ph.output_name not in component_spec.OUTPUTS:
raise ValueError(
'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.'
)
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(operator.add, [
self._recursively_encode(item, component_spec)
for item in ph.items
])
else:
raise TypeError(('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.').format(
type(ph),
str(placeholders.CommandlineArgumentType)))
def testExecPropertySimple(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('num_train_steps'), """
placeholder {
type: EXEC_PROPERTY
key: "num_train_steps"
}
""")
def testProtoOperatorDescriptor(self):
test_pb_filepath = os.path.join(os.path.dirname(__file__), 'testdata',
'proto_placeholder_operator.pbtxt')
with open(test_pb_filepath) as text_pb_file:
expected_pb = text_format.ParseLines(
text_pb_file, placeholder_pb2.PlaceholderExpression())
placeholder = ph.exec_property('splits_config').analyze[0]
component_spec = standard_component_specs.TransformSpec
self.assertProtoEquals(placeholder.encode(component_spec), expected_pb)
def testPlaceholdersInvolved(self):
p = ('google/' + ph.runtime_info('platform_config').user + '/' +
ph.output('model').uri + '/model/' + '0/' +
ph.exec_property('version'))
got = p.placeholders_involved()
got_dict = {type(x): x for x in got}
self.assertCountEqual(
{
ph.ArtifactPlaceholder, ph.ExecPropertyPlaceholder,
ph.RuntimeInfoPlaceholder
}, got_dict.keys())
def testBase64EncodeOperator(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('str_value').b64encode(), """
operator {
base64_encode_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "str_value"
}
}
}
}
""")
def testComplicatedConcat(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
'google/' + ph.output('model').uri + '/model/' + '0/' +
ph.exec_property('version'), """
operator {
concat_op {
expressions {
value {
string_value: "google/"
}
}
expressions {
operator {
artifact_uri_op {
expression {
operator {
index_op {
expression {
placeholder {
type: OUTPUT_ARTIFACT
key: "model"
}
}
index: 0
}
}
}
}
}
}
expressions {
value {
string_value: "/model/"
}
}
expressions {
value {
string_value: "0/"
}
}
expressions {
placeholder {
type: EXEC_PROPERTY
key: "version"
}
}
}
}
""")
def testProtoSerializationOperator(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('proto').serialize(
ph.ProtoSerializationFormat.JSON), """
operator {
proto_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "proto"
}
}
serialization_format: JSON
}
}
""")
def testExecPropertyListProtoSerialize(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('list_proto').serialize_list(
ph.ListSerializationFormat.JSON), """
operator {
list_serialization_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "list_proto"
}
}
serialization_format: JSON
}
}
""")
def testExecPropertyProtoField(self):
self._assert_placeholder_pb_equal(
ph.exec_property('proto')[0].a.b['c'], """
operator {
proto_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "proto"
}
}
proto_field_path: "[0]"
proto_field_path: ".a"
proto_field_path: ".b"
proto_field_path: "['c']"
}
}
""")
class HelloWorldComponent(BaseComponent):
"""Producer component."""
SPEC_CLASS = _HelloWorldSpec
EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec(
# TODO(b/143965964): move the image to private repo if the test is flaky
# due to docker hub.
image='gcr.io/google.com/cloudsdktool/cloud-sdk:latest',
command=['sh', '-c'],
args=[
'echo "hello ' + ph.exec_property('word') + '" | gsutil cp - ' +
ph.output('greeting')[0].uri
])
def __init__(self, word, greeting=None):
if not greeting:
artifact = standard_artifacts.String()
greeting = channel_utils.as_channel([artifact])
super().__init__(_HelloWorldSpec(word=word, greeting=greeting))
def testExecPropertyListProtoIndex(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.exec_property('list_proto')[0].serialize(
ph.ProtoSerializationFormat.JSON), """
operator {
proto_op {
expression {
operator {
index_op {
expression {
placeholder {
type: EXEC_PROPERTY
key: "list_proto"
}
}
index: 0
}
}
}
serialization_format: JSON
}
}
""")
image='google/cloud-sdk:278.0.0',
command=[
'sh',
'-exc',
'''
url="$0"
output_data_uri="$1"/data # TODO(b/150515270) Remove when fixed.
output_data_path=$(mktemp)
# Running the main code
wget "$0" -O "$output_data_path" || curl "$0" > "$output_data_path"
# Getting data out of the container
gsutil cp "$output_data_path" "$output_data_uri"
''',
ph.exec_property('url'),
ph.output('data')[0].uri,
],
)
grep_component = container_component.create_container_component(
name='FilterWithGrep',
inputs={
'text': standard_artifacts.ExternalArtifact,
},
outputs={
'filtered_text': standard_artifacts.ExternalArtifact,
},
parameters={
'pattern': str,
},
def testExecutionParameterTypeCheck(self):
int_parameter = ExecutionParameter(type=int)
int_parameter.type_check('int_parameter', 8)
with self.assertRaisesRegex(
TypeError, "Expected type <(class|type) 'int'>"
" for parameter u?'int_parameter'"):
int_parameter.type_check('int_parameter', 'string')
list_parameter = ExecutionParameter(type=List[int])
list_parameter.type_check('list_parameter', [])
list_parameter.type_check('list_parameter', [42])
with self.assertRaisesRegex(TypeError,
'Expecting a list for parameter'):
list_parameter.type_check('list_parameter', 42)
with self.assertRaisesRegex(
TypeError, "Expecting item type <(class|type) "
"'int'> for parameter u?'list_parameter'"):
list_parameter.type_check('list_parameter', [42, 'wrong item'])
dict_parameter = ExecutionParameter(type=Dict[str, int])
dict_parameter.type_check('dict_parameter', {})
dict_parameter.type_check('dict_parameter', {'key1': 1, 'key2': 2})
with self.assertRaisesRegex(TypeError,
'Expecting a dict for parameter'):
dict_parameter.type_check('dict_parameter', 'simple string')
with self.assertRaisesRegex(
TypeError, "Expecting value type "
"<(class|type) 'int'>"):
dict_parameter.type_check('dict_parameter', {'key1': '1'})
proto_parameter = ExecutionParameter(type=example_gen_pb2.Input)
proto_parameter.type_check('proto_parameter', example_gen_pb2.Input())
proto_parameter.type_check(
'proto_parameter',
proto_utils.proto_to_json(example_gen_pb2.Input()))
proto_parameter.type_check('proto_parameter',
{'splits': [{
'name': 'hello'
}]})
proto_parameter.type_check('proto_parameter', {'wrong_field': 42})
with self.assertRaisesRegex(
TypeError,
"Expected type <class 'tfx.proto.example_gen_pb2.Input'>"):
proto_parameter.type_check('proto_parameter', 42)
with self.assertRaises(json_format.ParseError):
proto_parameter.type_check('proto_parameter', {'splits': 42})
output_channel = channel.Channel(type=_OutputArtifact)
placeholder_parameter = ExecutionParameter(type=str)
placeholder_parameter.type_check(
'wrapped_channel_placeholder_parameter',
output_channel.future()[0].value)
placeholder_parameter.type_check(
'placeholder_parameter',
placeholder.runtime_info('platform_config').base_dir)
with self.assertRaisesRegex(
TypeError,
'Only simple RuntimeInfoPlaceholders are supported'):
placeholder_parameter.type_check(
'placeholder_parameter',
placeholder.runtime_info('platform_config').base_dir +
placeholder.exec_property('version'))
