def testConstruct(self): examples = standard_artifacts.Examples() binder = binder_component.DataViewBinder( input_examples=channel_utils.as_channel([examples]), data_view=channel_utils.as_channel([standard_artifacts.DataView() ])) self.assertIsNotNone(binder.outputs['output_examples'])
def __init__(self, create_decoder_func: Text, module_file: Optional[Text] = None, data_view: Optional[types.Channel] = None, instance_name: Optional[Text] = None): """Construct a StatisticsGen component. Args: create_decoder_func: If `module_file` is not None, this should be the name of the function in `module_file` that this component need to use to create the TfGraphRecordDecoder. Otherwise it should be the path (dot-delimited, e.g. "some_package.some_module.some_func") to such a function. The function must have the following signature: def create_decoder_func() -> tfx_bsl.coder.TfGraphRecordDecoder: ... module_file: The file path to a python module file, from which the function named after `create_decoder_func` will be loaded. If not provided, `create_decoder_func` is expected to be a path to a function. data_view: Output 'DataView' channel, in which a the decoder will be saved. instance_name: Optional unique instance name. Necessary iff multiple transform components are declared in the same pipeline. """ if data_view is None: data_view = types.Channel( type=standard_artifacts.DataView, artifacts=[standard_artifacts.DataView()]) spec = _TfGraphDataViewProviderSpec( module_file=module_file, create_decoder_func=create_decoder_func, data_view=data_view) super().__init__(spec=spec, instance_name=instance_name)
def testConstruct(self): examples = standard_artifacts.Examples() examples.split_names = artifact_utils.encode_split_names( ['train', 'eval']) examples.span = 1 binder = binder_component.DataViewBinder( input_examples=channel_utils.as_channel([examples]), data_view=channel_utils.as_channel([standard_artifacts.DataView() ])) output_examples = binder.outputs['output_examples'] self.assertIsNotNone(output_examples) output_examples = output_examples.get() self.assertLen(output_examples, 1) self._assert_example_artifact_equal(output_examples[0], examples)
def testExecutorModuleFileNotProvided(self): input_dict = {} output = standard_artifacts.DataView() output.uri = os.path.join(self._output_data_dir, 'output_data_view') output_dict = {'data_view': output} exec_properties = { 'create_decoder_func': '%s.%s' % (data_view_module.create_simple_decoder.__module__, data_view_module.create_simple_decoder.__name__), } executor = provider_executor.TfGraphDataViewProviderExecutor() executor.Do(input_dict, output_dict, exec_properties) loaded_decoder = tf_graph_record_decoder.load_decoder(output.uri) self.assertIsInstance(loaded_decoder, tf_graph_record_decoder.TFGraphRecordDecoder)
def testExecutorModuleFileProvided(self): input_dict = {} output = standard_artifacts.DataView() output.uri = os.path.join(self._output_data_dir, 'output_data_view') output_dict = {'data_view': [output]} exec_properties = { 'module_file': os.path.join(self._source_data_dir, 'module_file/data_view_module.py'), 'create_decoder_func': 'create_simple_decoder', } executor = provider_executor.TfGraphDataViewProviderExecutor() executor.Do(input_dict, output_dict, exec_properties) loaded_decoder = tf_graph_record_decoder.load_decoder(output.uri) self.assertIsInstance( loaded_decoder, tf_graph_record_decoder.LoadedDecoder)
def testDo(self): data_view = standard_artifacts.DataView() data_view.uri = '/old/data_view' data_view.id = 1 existing_custom_property = 'payload_format' input_examples = standard_artifacts.Examples() input_examples.uri = '/examples/1' input_examples.set_string_custom_property( existing_custom_property, 'VALUE1') input_dict = { 'input_examples': [input_examples], 'data_view': [data_view], } exec_properties = {} output_dict = { 'output_examples': [ standard_artifacts.Examples() ] } executor = binder_executor.DataViewBinderExecutor() executor.Do(input_dict, output_dict, exec_properties) output_examples = output_dict.get('output_examples') self.assertIsNotNone(output_examples) self.assertLen(output_examples, 1) oe = output_examples[0] self.assertEqual( oe.get_string_custom_property( binder_executor.DATA_VIEW_URI_PROPERTY_KEY), data_view.uri) self.assertEqual( oe.get_int_custom_property( binder_executor.DATA_VIEW_ID_PROPERTY_KEY), data_view.id) # output should share the URI with the input. self.assertEqual(oe.uri, input_examples.uri) # other custom properties should be inherited. self.assertEqual( oe.get_string_custom_property(existing_custom_property), input_examples.get_string_custom_property(existing_custom_property))
def test_raise_if_not_example(self): artifact = standard_artifacts.DataView() with self.assertRaisesRegex( AssertionError, 'must be of type standard_artifacts.Example'): tfxio_utils.get_tfxio_factory_from_artifact( artifact, _TELEMETRY_DESCRIPTORS)