def testRunWithIntermediateDeploymentConfig(self): self._pipeline.deployment_config.Pack(_LOCAL_DEPLOYMENT_CONFIG) beam_dag_runner.BeamDagRunner().run(self._pipeline) self.assertEqual( _component_executors, { 'my_example_gen': text_format.Parse( 'class_path: "tfx.components.example_gen_executor"', _PythonClassExecutableSpec()), 'my_transform': text_format.Parse( 'class_path: "tfx.components.transform_executor"', _PythonClassExecutableSpec()), 'my_trainer': text_format.Parse( 'class_path: "tfx.components.trainer_executor"', _PythonClassExecutableSpec()) }) self.assertEqual( _component_drivers, { 'my_example_gen': text_format.Parse( 'class_path: "tfx.components.example_gen_driver"', _PythonClassExecutableSpec()), 'my_transform': None, 'my_trainer': None }) self.assertEqual(_executed_components, ['my_example_gen', 'my_transform', 'my_trainer'])
def testDockerComponentLauncherInBeam(self): beam_dag_runner.BeamDagRunner().run( _create_pipeline(pipeline_name=self._pipeline_name, pipeline_root=self._pipeline_root, metadata_path=self._metadata_path, name='docker_e2e_test_in_beam')) metadata_config = metadata.sqlite_metadata_connection_config( self._metadata_path) with metadata.Metadata(metadata_config) as m: self.assertEqual(1, len(m.store.get_executions()))
def testRunWithIntermediateDeploymentConfig(self): self._pipeline.deployment_config.Pack(_LOCAL_DEPLOYMENT_CONFIG) beam_dag_runner.BeamDagRunner._PIPELINE_NODE_DO_FN_CLS = _FakeComponentAsDoFn beam_dag_runner.BeamDagRunner().run(self._pipeline) self.assertEqual( _component_executors, { 'my_example_gen': text_format.Parse( 'class_path: "tfx.components.example_gen_executor"', _PythonClassExecutableSpec()), 'my_transform': text_format.Parse( 'class_path: "tfx.components.transform_executor"', _PythonClassExecutableSpec()), 'my_trainer': text_format.Parse( 'class_path: "tfx.components.trainer_executor"', _PythonClassExecutableSpec()), 'my_importer': None, }) self.assertEqual( _component_drivers, { 'my_example_gen': text_format.Parse( 'class_path: "tfx.components.example_gen_driver"', _PythonClassExecutableSpec()), 'my_transform': None, 'my_trainer': None, 'my_importer': None, }) # 'my_importer' has no upstream and can be executed in any order. self.assertIn('my_importer', _executed_components) _executed_components.remove('my_importer') self.assertEqual(_executed_components, ['my_example_gen', 'my_transform', 'my_trainer']) # Verifies that every component gets a not-None pipeline_run. self.assertTrue(all(_conponent_to_pipeline_run.values()))
def testRun(self): beam_dag_runner.BeamDagRunner().run(self._pipeline) self.assertEqual(_executed_components, ['my_example_gen', 'my_transform', 'my_trainer'])