def testConcatUriWithString(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.output('model').uri + '/model', """ operator { concat_op { expressions { operator { artifact_uri_op { expression { operator { index_op { expression { placeholder { type: OUTPUT_ARTIFACT key: "model" } } } } } } } } expressions { value { string_value: "/model" } } } } """)
def _recursively_encode( self, ph: Union[placeholders.CommandlineArgumentType, placeholder.Placeholder, str], component_spec: Optional[types.ComponentSpec] = None ) -> Union[str, placeholder.Placeholder]: """This method recursively encodes placeholders.CommandlineArgumentType. The recursion ending condision is that the input ph is alerady a string or placeholder.Placeholder. Args: ph: The placeholder to encode. component_spec: Optional. The ComponentSpec to help with the encoding. Returns: The encoded placeholder in the type of string or placeholder.Placeholder. """ if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder): # If there is no place holder. Or if the placeholder is already a # new style placeholder. # No further encoding is needed. return cast(Union[str, placeholder.Placeholder], ph) elif isinstance(ph, placeholders.InputValuePlaceholder): if not component_spec: raise ValueError( 'Requires component spec to encode InputValuePlaceholder.') if ph.input_name in component_spec.INPUTS: return placeholder.input(ph.input_name)[0].value elif ph.input_name in component_spec.PARAMETERS: return placeholder.exec_property(ph.input_name) else: raise ValueError( 'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.' ) elif isinstance(ph, placeholders.InputUriPlaceholder): if component_spec and ph.input_name not in component_spec.INPUTS: raise ValueError( 'For InputUriPlaceholder, input name must be in component\'s INPUTS.' ) return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): if component_spec and ph.output_name not in component_spec.OUTPUTS: raise ValueError( 'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.' ) return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce(operator.add, [ self._recursively_encode(item, component_spec) for item in ph.items ]) else: raise TypeError(('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.').format( type(ph), str(placeholders.CommandlineArgumentType)))
def testPlaceholdersInvolved(self): p = ('google/' + ph.runtime_info('platform_config').user + '/' + ph.output('model').uri + '/model/' + '0/' + ph.exec_property('version')) got = p.placeholders_involved() got_dict = {type(x): x for x in got} self.assertCountEqual( { ph.ArtifactPlaceholder, ph.ExecPropertyPlaceholder, ph.RuntimeInfoPlaceholder }, got_dict.keys())
def testComplicatedConcat(self): self._assert_placeholder_pb_equal_and_deepcopyable( 'google/' + ph.output('model').uri + '/model/' + '0/' + ph.exec_property('version'), """ operator { concat_op { expressions { value { string_value: "google/" } } expressions { operator { artifact_uri_op { expression { operator { index_op { expression { placeholder { type: OUTPUT_ARTIFACT key: "model" } } index: 0 } } } } } } expressions { value { string_value: "/model/" } } expressions { value { string_value: "0/" } } expressions { placeholder { type: EXEC_PROPERTY key: "version" } } } } """)
class HelloWorldComponent(BaseComponent): """Producer component.""" SPEC_CLASS = _HelloWorldSpec EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec( # TODO(b/143965964): move the image to private repo if the test is flaky # due to docker hub. image='gcr.io/google.com/cloudsdktool/cloud-sdk:latest', command=['sh', '-c'], args=[ 'echo "hello ' + ph.exec_property('word') + '" | gsutil cp - ' + ph.output('greeting')[0].uri ]) def __init__(self, word, greeting=None): if not greeting: artifact = standard_artifacts.String() greeting = channel_utils.as_channel([artifact]) super().__init__(_HelloWorldSpec(word=word, greeting=greeting))
def _recursively_encode( self, ph: placeholders.CommandlineArgumentType ) -> Union[str, placeholder.Placeholder]: if isinstance(ph, str): return ph elif isinstance(ph, placeholders.InputValuePlaceholder): return placeholder.input(ph.input_name)[0] elif isinstance(ph, placeholders.InputUriPlaceholder): return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce( operator.add, [self._recursively_encode(item) for item in ph.items]) else: raise TypeError(('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.').format( type(ph), str(placeholders.CommandlineArgumentType)))
def _recursively_encode( self, ph: Union[placeholders.CommandlineArgumentType, placeholder.Placeholder, str] ) -> Union[str, placeholder.Placeholder]: """This method recursively encodes placeholders.CommandlineArgumentType. The recursion ending condision is that the input ph is alerady a string or placeholder.Placeholder. Args: ph: the placeholder to encode. Returns: The encoded placeholder in the type of string or placeholder.Placeholder. """ if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder): # If there is no place holder. Or if the placeholder is already a # new style placeholder. # No further encoding is needed. return cast(Union[str, placeholder.Placeholder], ph) elif isinstance(ph, placeholders.InputValuePlaceholder): return placeholder.input(ph.input_name)[0] elif isinstance(ph, placeholders.InputUriPlaceholder): return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce( operator.add, [self._recursively_encode(item) for item in ph.items]) else: raise TypeError( ('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.') .format(type(ph), str(placeholders.CommandlineArgumentType)))
command=[ 'sh', '-exc', ''' url="$0" output_data_uri="$1"/data # TODO(b/150515270) Remove when fixed. output_data_path=$(mktemp) # Running the main code wget "$0" -O "$output_data_path" || curl "$0" > "$output_data_path" # Getting data out of the container gsutil cp "$output_data_path" "$output_data_uri" ''', ph.exec_property('url'), ph.output('data')[0].uri, ], ) grep_component = container_component.create_container_component( name='FilterWithGrep', inputs={ 'text': standard_artifacts.ExternalArtifact, }, outputs={ 'filtered_text': standard_artifacts.ExternalArtifact, }, parameters={ 'pattern': str, }, # The component code uses gsutil to upload the data to GCS, so the