def testConcatUriWithString(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.output('model').uri + '/model', """
operator {
concat_op {
expressions {
operator {
artifact_uri_op {
expression {
operator {
index_op {
expression {
placeholder {
type: OUTPUT_ARTIFACT
key: "model"
}
}
}
}
}
}
}
}
expressions {
value {
string_value: "/model"
}
}
}
}
""")
def _recursively_encode(
self,
ph: Union[placeholders.CommandlineArgumentType,
placeholder.Placeholder, str],
component_spec: Optional[types.ComponentSpec] = None
) -> Union[str, placeholder.Placeholder]:
"""This method recursively encodes placeholders.CommandlineArgumentType.
The recursion ending condision is that the input ph is alerady a string
or placeholder.Placeholder.
Args:
ph: The placeholder to encode.
component_spec: Optional. The ComponentSpec to help with the encoding.
Returns:
The encoded placeholder in the type of string or placeholder.Placeholder.
"""
if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder):
# If there is no place holder. Or if the placeholder is already a
# new style placeholder.
# No further encoding is needed.
return cast(Union[str, placeholder.Placeholder], ph)
elif isinstance(ph, placeholders.InputValuePlaceholder):
if not component_spec:
raise ValueError(
'Requires component spec to encode InputValuePlaceholder.')
if ph.input_name in component_spec.INPUTS:
return placeholder.input(ph.input_name)[0].value
elif ph.input_name in component_spec.PARAMETERS:
return placeholder.exec_property(ph.input_name)
else:
raise ValueError(
'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.'
)
elif isinstance(ph, placeholders.InputUriPlaceholder):
if component_spec and ph.input_name not in component_spec.INPUTS:
raise ValueError(
'For InputUriPlaceholder, input name must be in component\'s INPUTS.'
)
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
if component_spec and ph.output_name not in component_spec.OUTPUTS:
raise ValueError(
'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.'
)
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(operator.add, [
self._recursively_encode(item, component_spec)
for item in ph.items
])
else:
raise TypeError(('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.').format(
type(ph),
str(placeholders.CommandlineArgumentType)))
def testPlaceholdersInvolved(self):
p = ('google/' + ph.runtime_info('platform_config').user + '/' +
ph.output('model').uri + '/model/' + '0/' +
ph.exec_property('version'))
got = p.placeholders_involved()
got_dict = {type(x): x for x in got}
self.assertCountEqual(
{
ph.ArtifactPlaceholder, ph.ExecPropertyPlaceholder,
ph.RuntimeInfoPlaceholder
}, got_dict.keys())
def testComplicatedConcat(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
'google/' + ph.output('model').uri + '/model/' + '0/' +
ph.exec_property('version'), """
operator {
concat_op {
expressions {
value {
string_value: "google/"
}
}
expressions {
operator {
artifact_uri_op {
expression {
operator {
index_op {
expression {
placeholder {
type: OUTPUT_ARTIFACT
key: "model"
}
}
index: 0
}
}
}
}
}
}
expressions {
value {
string_value: "/model/"
}
}
expressions {
value {
string_value: "0/"
}
}
expressions {
placeholder {
type: EXEC_PROPERTY
key: "version"
}
}
}
}
""")
class HelloWorldComponent(BaseComponent):
"""Producer component."""
SPEC_CLASS = _HelloWorldSpec
EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec(
# TODO(b/143965964): move the image to private repo if the test is flaky
# due to docker hub.
image='gcr.io/google.com/cloudsdktool/cloud-sdk:latest',
command=['sh', '-c'],
args=[
'echo "hello ' + ph.exec_property('word') + '" | gsutil cp - ' +
ph.output('greeting')[0].uri
])
def __init__(self, word, greeting=None):
if not greeting:
artifact = standard_artifacts.String()
greeting = channel_utils.as_channel([artifact])
super().__init__(_HelloWorldSpec(word=word, greeting=greeting))
def _recursively_encode(
self, ph: placeholders.CommandlineArgumentType
) -> Union[str, placeholder.Placeholder]:
if isinstance(ph, str):
return ph
elif isinstance(ph, placeholders.InputValuePlaceholder):
return placeholder.input(ph.input_name)[0]
elif isinstance(ph, placeholders.InputUriPlaceholder):
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(
operator.add,
[self._recursively_encode(item) for item in ph.items])
else:
raise TypeError(('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.').format(
type(ph),
str(placeholders.CommandlineArgumentType)))
def _recursively_encode(
self, ph: Union[placeholders.CommandlineArgumentType,
placeholder.Placeholder, str]
) -> Union[str, placeholder.Placeholder]:
"""This method recursively encodes placeholders.CommandlineArgumentType.
The recursion ending condision is that the input ph is alerady a string
or placeholder.Placeholder.
Args:
ph: the placeholder to encode.
Returns:
The encoded placeholder in the type of string or placeholder.Placeholder.
"""
if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder):
# If there is no place holder. Or if the placeholder is already a
# new style placeholder.
# No further encoding is needed.
return cast(Union[str, placeholder.Placeholder], ph)
elif isinstance(ph, placeholders.InputValuePlaceholder):
return placeholder.input(ph.input_name)[0]
elif isinstance(ph, placeholders.InputUriPlaceholder):
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(
operator.add,
[self._recursively_encode(item) for item in ph.items])
else:
raise TypeError(
('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.')
.format(type(ph), str(placeholders.CommandlineArgumentType)))
command=[
'sh',
'-exc',
'''
url="$0"
output_data_uri="$1"/data # TODO(b/150515270) Remove when fixed.
output_data_path=$(mktemp)
# Running the main code
wget "$0" -O "$output_data_path" || curl "$0" > "$output_data_path"
# Getting data out of the container
gsutil cp "$output_data_path" "$output_data_uri"
''',
ph.exec_property('url'),
ph.output('data')[0].uri,
],
)
grep_component = container_component.create_container_component(
name='FilterWithGrep',
inputs={
'text': standard_artifacts.ExternalArtifact,
},
outputs={
'filtered_text': standard_artifacts.ExternalArtifact,
},
parameters={
'pattern': str,
},
# The component code uses gsutil to upload the data to GCS, so the