def _recursively_encode( self, ph: Union[placeholders.CommandlineArgumentType, placeholder.Placeholder, str], component_spec: Optional[types.ComponentSpec] = None ) -> Union[str, placeholder.Placeholder]: """This method recursively encodes placeholders.CommandlineArgumentType. The recursion ending condision is that the input ph is alerady a string or placeholder.Placeholder. Args: ph: The placeholder to encode. component_spec: Optional. The ComponentSpec to help with the encoding. Returns: The encoded placeholder in the type of string or placeholder.Placeholder. """ if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder): # If there is no place holder. Or if the placeholder is already a # new style placeholder. # No further encoding is needed. return cast(Union[str, placeholder.Placeholder], ph) elif isinstance(ph, placeholders.InputValuePlaceholder): if not component_spec: raise ValueError( 'Requires component spec to encode InputValuePlaceholder.') if ph.input_name in component_spec.INPUTS: return placeholder.input(ph.input_name)[0].value elif ph.input_name in component_spec.PARAMETERS: return placeholder.exec_property(ph.input_name) else: raise ValueError( 'For InputValuePlaceholder, input name must be in component\'s INPUTS or PARAMETERS.' ) elif isinstance(ph, placeholders.InputUriPlaceholder): if component_spec and ph.input_name not in component_spec.INPUTS: raise ValueError( 'For InputUriPlaceholder, input name must be in component\'s INPUTS.' ) return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): if component_spec and ph.output_name not in component_spec.OUTPUTS: raise ValueError( 'For OutputUriPlaceholder, output name must be in component\'s OUTPUTS.' ) return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce(operator.add, [ self._recursively_encode(item, component_spec) for item in ph.items ]) else: raise TypeError(('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.').format( type(ph), str(placeholders.CommandlineArgumentType)))
class ByeWorldComponent(BaseComponent): """Consumer component.""" SPEC_CLASS = _ByeWorldSpec EXECUTOR_SPEC = executor_specs.TemplatedExecutorContainerSpec( image='bash:latest', command=['echo'], args=['received ' + ph.input('hearing')[0].value]) def __init__(self, hearing): super(ByeWorldComponent, self).__init__(_ByeWorldSpec(hearing=hearing))
def _recursively_encode( self, ph: placeholders.CommandlineArgumentType ) -> Union[str, placeholder.Placeholder]: if isinstance(ph, str): return ph elif isinstance(ph, placeholders.InputValuePlaceholder): return placeholder.input(ph.input_name)[0] elif isinstance(ph, placeholders.InputUriPlaceholder): return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce( operator.add, [self._recursively_encode(item) for item in ph.items]) else: raise TypeError(('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.').format( type(ph), str(placeholders.CommandlineArgumentType)))
def testPrimitiveArtifactValue(self): self._assert_placeholder_pb_equal( ph.input('primitive').value, """ operator { artifact_value_op { expression { placeholder { type: INPUT_ARTIFACT key: "primitive" } } } } """)
def testArtifactUriSimple(self): self._assert_placeholder_pb_equal( ph.input('model').uri, """ operator { artifact_uri_op { expression { placeholder { type: INPUT_ARTIFACT key: "model" } } } } """)
def _recursively_encode( self, ph: Union[placeholders.CommandlineArgumentType, placeholder.Placeholder, str] ) -> Union[str, placeholder.Placeholder]: """This method recursively encodes placeholders.CommandlineArgumentType. The recursion ending condision is that the input ph is alerady a string or placeholder.Placeholder. Args: ph: the placeholder to encode. Returns: The encoded placeholder in the type of string or placeholder.Placeholder. """ if isinstance(ph, str) or isinstance(ph, placeholder.Placeholder): # If there is no place holder. Or if the placeholder is already a # new style placeholder. # No further encoding is needed. return cast(Union[str, placeholder.Placeholder], ph) elif isinstance(ph, placeholders.InputValuePlaceholder): return placeholder.input(ph.input_name)[0] elif isinstance(ph, placeholders.InputUriPlaceholder): return placeholder.input(ph.input_name)[0].uri elif isinstance(ph, placeholders.OutputUriPlaceholder): return placeholder.output(ph.output_name)[0].uri elif isinstance(ph, placeholders.ConcatPlaceholder): # operator.add wil use the overloaded __add__ operator for Placeholder # instances. return functools.reduce( operator.add, [self._recursively_encode(item) for item in ph.items]) else: raise TypeError( ('Unsupported type of placeholder arguments: "{}".' ' Supported types are {}.') .format(type(ph), str(placeholders.CommandlineArgumentType)))
def testArtifactUriWithIndex(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.input('model')[0].uri, """ operator { artifact_uri_op { expression { operator { index_op { expression { placeholder { type: INPUT_ARTIFACT key: "model" } } index: 0 } } } } } """)
def testArtifactProperty(self): self._assert_placeholder_pb_equal_and_deepcopyable( ph.input('model')[0].property('blessed'), """ operator { artifact_property_op { expression { operator { index_op { expression { placeholder { type: INPUT_ARTIFACT key: "model" } } index: 0 } } } key: "blessed" } } """)
def testArtifactSplitUriWithIndex(self): self._assert_placeholder_pb_equal( ph.input('model')[0].split_uri('train'), """ operator { artifact_uri_op { expression { operator { index_op { expression { placeholder { type: INPUT_ARTIFACT key: "model" } } index: 0 } } } split: "train" } } """)
text_uri="$1"/data # TODO(b/150515270) Remove when fixed. text_path=$(mktemp) filtered_text_uri="$2"/data # TODO(b/150515270) Remove when fixed. filtered_text_path=$(mktemp) # Getting data into the container gsutil cp "$text_uri" "$text_path" # Running the main code grep "$pattern" "$text_path" >"$filtered_text_path" # Getting data out of the container gsutil cp "$filtered_text_path" "$filtered_text_uri" ''', ph.exec_property('pattern'), ph.input('text')[0].uri, ph.output('filtered_text')[0].uri, ], ) print_component = container_component.create_container_component( name='Print', inputs={ 'text': standard_artifacts.ExternalArtifact, }, # The component code uses gsutil to upload the data to GCS, so the # container image needs to have gsutil installed and configured. # Fixing b/150670779 by merging cl/294536017 will lift this limitation. image='google/cloud-sdk:278.0.0', command=[ 'sh',
def testJsonSerializable(self): json_text = json_utils.dumps(ph.input('model').uri) python_instance = json_utils.loads(json_text) self.assertEqual( ph.input('model').uri.encode(), python_instance.encode())