def testExecPropertyProtoField(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('proto')[0].a.b['c'], """ operator { proto_op { expression { operator { index_op { expression { placeholder { type: EXEC_PROPERTY key: "proto" } } index: 0 } } } proto_field_path: ".a" proto_field_path: ".b" proto_field_path: "['c']" } } """) self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('proto').a['b'].c[1], """ operator { index_op { expression { operator { proto_op { expression { placeholder { type: EXEC_PROPERTY key: "proto" } } proto_field_path: ".a" proto_field_path: "['b']" proto_field_path: ".c" } } } index: 1 } } """)
def _recursively_encode( self, ph: Union[placeholders.CommandlineArgumentType, placeholder.Placeholder, str], component_spec: Optional[types.ComponentSpec] = None ) -> Union[str, placeholder.Placeholder]: """This method recursively encodes placeholders.CommandlineArgumentType. The recursion ending condision is that the input ph is alerady a string or placeholder.Placeholder. Args: ph: The placeholder to encode. component_spec: Optional. The ComponentSpec to help with the encoding. Returns: The encoded placeholder in the type of string or placeholder.Placeholder. """ if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder): # If there is no place holder. Or if the placeholder is already a # new style placeholder. # No further encoding is needed. return cast(Union[str, placeholder.Placeholder], ph) elif isinstance(ph, placeholders.InputValuePlaceholder): if not component_spec: raise ValueError( 'Requires component spec to encode InputValuePlaceholder.') if ph.input_name in component_spec.INPUTS: return placeholder.input(ph.input_name)[0].value elif ph.input_name in component_spec.PARAMETERS: return placeholder.exec_property(ph.input_name) else: raise ValueError( 'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.' ) elif isinstance(ph, placeholders.InputUriPlaceholder): if component_spec and ph.input_name not in component_spec.INPUTS: raise ValueError( 'For InputUriPlaceholder, input name must be in component\'s INPUTS.' ) return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): if component_spec and ph.output_name not in component_spec.OUTPUTS: raise ValueError( 'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.' ) return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce(operator.add, [ self._recursively_encode(item, component_spec) for item in ph.items ]) else: raise TypeError(('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.').format( type(ph), str(placeholders.CommandlineArgumentType)))
def testExecPropertySimple(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('num_train_steps'), """ placeholder { type: EXEC_PROPERTY key: "num_train_steps" } """)
def testProtoOperatorDescriptor(self): test_pb_filepath = os.path.join(os.path.dirname(__file__), 'testdata', 'proto_placeholder_operator.pbtxt') with open(test_pb_filepath) as text_pb_file: expected_pb = text_format.ParseLines( text_pb_file, placeholder_pb2.PlaceholderExpression()) placeholder = ph.exec_property('splits_config').analyze[0] component_spec = standard_component_specs.TransformSpec self.assertProtoEquals(placeholder.encode(component_spec), expected_pb)
def testPlaceholdersInvolved(self): p = ('google/' + ph.runtime_info('platform_config').user + '/' + ph.output('model').uri + '/model/' + '0/' + ph.exec_property('version')) got = p.placeholders_involved() got_dict = {type(x): x for x in got} self.assertCountEqual( { ph.ArtifactPlaceholder, ph.ExecPropertyPlaceholder, ph.RuntimeInfoPlaceholder }, got_dict.keys())
def testBase64EncodeOperator(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('str_value').b64encode(), """ operator { base64_encode_op { expression { placeholder { type: EXEC_PROPERTY key: "str_value" } } } } """)
def testComplicatedConcat(self): self._assert_placeholder_pb_equal_and_deepcopyable( 'google/' + ph.output('model').uri + '/model/' + '0/' + ph.exec_property('version'), """ operator { concat_op { expressions { value { string_value: "google/" } } expressions { operator { artifact_uri_op { expression { operator { index_op { expression { placeholder { type: OUTPUT_ARTIFACT key: "model" } } index: 0 } } } } } } expressions { value { string_value: "/model/" } } expressions { value { string_value: "0/" } } expressions { placeholder { type: EXEC_PROPERTY key: "version" } } } } """)
def testProtoSerializationOperator(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('proto').serialize( ph.ProtoSerializationFormat.JSON), """ operator { proto_op { expression { placeholder { type: EXEC_PROPERTY key: "proto" } } serialization_format: JSON } } """)
def testExecPropertyListProtoSerialize(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('list_proto').serialize_list( ph.ListSerializationFormat.JSON), """ operator { list_serialization_op { expression { placeholder { type: EXEC_PROPERTY key: "list_proto" } } serialization_format: JSON } } """)
def testExecPropertyProtoField(self): self._assert_placeholder_pb_equal( ph.exec_property('proto')[0].a.b['c'], """ operator { proto_op { expression { placeholder { type: EXEC_PROPERTY key: "proto" } } proto_field_path: "[0]" proto_field_path: ".a" proto_field_path: ".b" proto_field_path: "['c']" } } """)
class HelloWorldComponent(BaseComponent): """Producer component.""" SPEC_CLASS = _HelloWorldSpec EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec( # TODO(b/143965964): move the image to private repo if the test is flaky # due to docker hub. image='gcr.io/google.com/cloudsdktool/cloud-sdk:latest', command=['sh', '-c'], args=[ 'echo "hello ' + ph.exec_property('word') + '" | gsutil cp - ' + ph.output('greeting')[0].uri ]) def __init__(self, word, greeting=None): if not greeting: artifact = standard_artifacts.String() greeting = channel_utils.as_channel([artifact]) super().__init__(_HelloWorldSpec(word=word, greeting=greeting))
def testExecPropertyListProtoIndex(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.exec_property('list_proto')[0].serialize( ph.ProtoSerializationFormat.JSON), """ operator { proto_op { expression { operator { index_op { expression { placeholder { type: EXEC_PROPERTY key: "list_proto" } } index: 0 } } } serialization_format: JSON } } """)
image='google/cloud-sdk:278.0.0', command=[ 'sh', '-exc', ''' url="$0" output_data_uri="$1"/data # TODO(b/150515270) Remove when fixed. output_data_path=$(mktemp) # Running the main code wget "$0" -O "$output_data_path" || curl "$0" > "$output_data_path" # Getting data out of the container gsutil cp "$output_data_path" "$output_data_uri" ''', ph.exec_property('url'), ph.output('data')[0].uri, ], ) grep_component = container_component.create_container_component( name='FilterWithGrep', inputs={ 'text': standard_artifacts.ExternalArtifact, }, outputs={ 'filtered_text': standard_artifacts.ExternalArtifact, }, parameters={ 'pattern': str, },
def testExecutionParameterTypeCheck(self): int_parameter = ExecutionParameter(type=int) int_parameter.type_check('int_parameter', 8) with self.assertRaisesRegex( TypeError, "Expected type <(class|type) 'int'>" " for parameter u?'int_parameter'"): int_parameter.type_check('int_parameter', 'string') list_parameter = ExecutionParameter(type=List[int]) list_parameter.type_check('list_parameter', []) list_parameter.type_check('list_parameter', [42]) with self.assertRaisesRegex(TypeError, 'Expecting a list for parameter'): list_parameter.type_check('list_parameter', 42) with self.assertRaisesRegex( TypeError, "Expecting item type <(class|type) " "'int'> for parameter u?'list_parameter'"): list_parameter.type_check('list_parameter', [42, 'wrong item']) dict_parameter = ExecutionParameter(type=Dict[str, int]) dict_parameter.type_check('dict_parameter', {}) dict_parameter.type_check('dict_parameter', {'key1': 1, 'key2': 2}) with self.assertRaisesRegex(TypeError, 'Expecting a dict for parameter'): dict_parameter.type_check('dict_parameter', 'simple string') with self.assertRaisesRegex( TypeError, "Expecting value type " "<(class|type) 'int'>"): dict_parameter.type_check('dict_parameter', {'key1': '1'}) proto_parameter = ExecutionParameter(type=example_gen_pb2.Input) proto_parameter.type_check('proto_parameter', example_gen_pb2.Input()) proto_parameter.type_check( 'proto_parameter', proto_utils.proto_to_json(example_gen_pb2.Input())) proto_parameter.type_check('proto_parameter', {'splits': [{ 'name': 'hello' }]}) proto_parameter.type_check('proto_parameter', {'wrong_field': 42}) with self.assertRaisesRegex( TypeError, "Expected type <class 'tfx.proto.example_gen_pb2.Input'>"): proto_parameter.type_check('proto_parameter', 42) with self.assertRaises(json_format.ParseError): proto_parameter.type_check('proto_parameter', {'splits': 42}) output_channel = channel.Channel(type=_OutputArtifact) placeholder_parameter = ExecutionParameter(type=str) placeholder_parameter.type_check( 'wrapped_channel_placeholder_parameter', output_channel.future()[0].value) placeholder_parameter.type_check( 'placeholder_parameter', placeholder.runtime_info('platform_config').base_dir) with self.assertRaisesRegex( TypeError, 'Only simple RuntimeInfoPlaceholders are supported'): placeholder_parameter.type_check( 'placeholder_parameter', placeholder.runtime_info('platform_config').base_dir + placeholder.exec_property('version'))