def _recursively_encode(
self,
ph: Union[placeholders.CommandlineArgumentType,
placeholder.Placeholder, str],
component_spec: Optional[types.ComponentSpec] = None
) -> Union[str, placeholder.Placeholder]:
"""This method recursively encodes placeholders.CommandlineArgumentType.
The recursion ending condision is that the input ph is alerady a string
or placeholder.Placeholder.
Args:
ph: The placeholder to encode.
component_spec: Optional. The ComponentSpec to help with the encoding.
Returns:
The encoded placeholder in the type of string or placeholder.Placeholder.
"""
if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder):
# If there is no place holder. Or if the placeholder is already a
# new style placeholder.
# No further encoding is needed.
return cast(Union[str, placeholder.Placeholder], ph)
elif isinstance(ph, placeholders.InputValuePlaceholder):
if not component_spec:
raise ValueError(
'Requires component spec to encode InputValuePlaceholder.')
if ph.input_name in component_spec.INPUTS:
return placeholder.input(ph.input_name)[0].value
elif ph.input_name in component_spec.PARAMETERS:
return placeholder.exec_property(ph.input_name)
else:
raise ValueError(
'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.'
)
elif isinstance(ph, placeholders.InputUriPlaceholder):
if component_spec and ph.input_name not in component_spec.INPUTS:
raise ValueError(
'For InputUriPlaceholder, input name must be in component\'s INPUTS.'
)
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
if component_spec and ph.output_name not in component_spec.OUTPUTS:
raise ValueError(
'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.'
)
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(operator.add, [
self._recursively_encode(item, component_spec)
for item in ph.items
])
else:
raise TypeError(('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.').format(
type(ph),
str(placeholders.CommandlineArgumentType)))
class ByeWorldComponent(BaseComponent):
"""Consumer component."""
SPEC_CLASS = _ByeWorldSpec
EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec(
image='bash:latest',
command=['echo'],
args=['received ' + ph.input('hearing')[0].value])
def __init__(self, hearing):
super(ByeWorldComponent, self).__init__(_ByeWorldSpec(hearing=hearing))
def _recursively_encode(
self, ph: placeholders.CommandlineArgumentType
) -> Union[str, placeholder.Placeholder]:
if isinstance(ph, str):
return ph
elif isinstance(ph, placeholders.InputValuePlaceholder):
return placeholder.input(ph.input_name)[0]
elif isinstance(ph, placeholders.InputUriPlaceholder):
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(
operator.add,
[self._recursively_encode(item) for item in ph.items])
else:
raise TypeError(('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.').format(
type(ph),
str(placeholders.CommandlineArgumentType)))
def testPrimitiveArtifactValue(self):
self._assert_placeholder_pb_equal(
ph.input('primitive').value, """
operator {
artifact_value_op {
expression {
placeholder {
type: INPUT_ARTIFACT
key: "primitive"
}
}
}
}
""")
def testArtifactUriSimple(self):
self._assert_placeholder_pb_equal(
ph.input('model').uri, """
operator {
artifact_uri_op {
expression {
placeholder {
type: INPUT_ARTIFACT
key: "model"
}
}
}
}
""")
def _recursively_encode(
self, ph: Union[placeholders.CommandlineArgumentType,
placeholder.Placeholder, str]
) -> Union[str, placeholder.Placeholder]:
"""This method recursively encodes placeholders.CommandlineArgumentType.
The recursion ending condision is that the input ph is alerady a string
or placeholder.Placeholder.
Args:
ph: the placeholder to encode.
Returns:
The encoded placeholder in the type of string or placeholder.Placeholder.
"""
if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder):
# If there is no place holder. Or if the placeholder is already a
# new style placeholder.
# No further encoding is needed.
return cast(Union[str, placeholder.Placeholder], ph)
elif isinstance(ph, placeholders.InputValuePlaceholder):
return placeholder.input(ph.input_name)[0]
elif isinstance(ph, placeholders.InputUriPlaceholder):
return placeholder.input(ph.input_name)[0].uri
elif isinstance(ph, placeholders.OutputUriPlaceholder):
return placeholder.output(ph.output_name)[0].uri
elif isinstance(ph, placeholders.ConcatPlaceholder):
# operator.add wil use the overloaded __add__ operator for Placeholder
# instances.
return functools.reduce(
operator.add,
[self._recursively_encode(item) for item in ph.items])
else:
raise TypeError(
('Unsupported type of placeholder arguments: "{}".'
' Supported types are {}.')
.format(type(ph), str(placeholders.CommandlineArgumentType)))
def testArtifactUriWithIndex(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.input('model')[0].uri, """
operator {
artifact_uri_op {
expression {
operator {
index_op {
expression {
placeholder {
type: INPUT_ARTIFACT
key: "model"
}
}
index: 0
}
}
}
}
}
""")
def testArtifactProperty(self):
self._assert_placeholder_pb_equal_and_deepcopyable(
ph.input('model')[0].property('blessed'), """
operator {
artifact_property_op {
expression {
operator {
index_op {
expression {
placeholder {
type: INPUT_ARTIFACT
key: "model"
}
}
index: 0
}
}
}
key: "blessed"
}
}
""")
def testArtifactSplitUriWithIndex(self):
self._assert_placeholder_pb_equal(
ph.input('model')[0].split_uri('train'), """
operator {
artifact_uri_op {
expression {
operator {
index_op {
expression {
placeholder {
type: INPUT_ARTIFACT
key: "model"
}
}
index: 0
}
}
}
split: "train"
}
}
""")
text_uri="$1"/data # TODO(b/150515270) Remove when fixed.
text_path=$(mktemp)
filtered_text_uri="$2"/data # TODO(b/150515270) Remove when fixed.
filtered_text_path=$(mktemp)
# Getting data into the container
gsutil cp "$text_uri" "$text_path"
# Running the main code
grep "$pattern" "$text_path" >"$filtered_text_path"
# Getting data out of the container
gsutil cp "$filtered_text_path" "$filtered_text_uri"
''',
ph.exec_property('pattern'),
ph.input('text')[0].uri,
ph.output('filtered_text')[0].uri,
],
)
print_component = container_component.create_container_component(
name='Print',
inputs={
'text': standard_artifacts.ExternalArtifact,
},
# The component code uses gsutil to upload the data to GCS, so the
# container image needs to have gsutil installed and configured.
# Fixing b/150670779 by merging cl/294536017 will lift this limitation.
image='google/cloud-sdk:278.0.0',
command=[
'sh',
def testJsonSerializable(self):
json_text = json_utils.dumps(ph.input('model').uri)
python_instance = json_utils.loads(json_text)
self.assertEqual(
ph.input('model').uri.encode(), python_instance.encode())
