def testRunWithIntermediateDeploymentConfig(self):
self._pipeline.deployment_config.Pack(_LOCAL_DEPLOYMENT_CONFIG)
beam_dag_runner.BeamDagRunner().run(self._pipeline)
self.assertEqual(
_component_executors, {
'my_example_gen':
text_format.Parse(
'class_path: "tfx.components.example_gen_executor"',
_PythonClassExecutableSpec()),
'my_transform':
text_format.Parse(
'class_path: "tfx.components.transform_executor"',
_PythonClassExecutableSpec()),
'my_trainer':
text_format.Parse(
'class_path: "tfx.components.trainer_executor"',
_PythonClassExecutableSpec())
})
self.assertEqual(
_component_drivers, {
'my_example_gen':
text_format.Parse(
'class_path: "tfx.components.example_gen_driver"',
_PythonClassExecutableSpec()),
'my_transform':
None,
'my_trainer':
None
})
self.assertEqual(_executed_components,
['my_example_gen', 'my_transform', 'my_trainer'])
def testDockerComponentLauncherInBeam(self):
beam_dag_runner.BeamDagRunner().run(
_create_pipeline(pipeline_name=self._pipeline_name,
pipeline_root=self._pipeline_root,
metadata_path=self._metadata_path,
name='docker_e2e_test_in_beam'))
metadata_config = metadata.sqlite_metadata_connection_config(
self._metadata_path)
with metadata.Metadata(metadata_config) as m:
self.assertEqual(1, len(m.store.get_executions()))
def testRunWithIntermediateDeploymentConfig(self):
self._pipeline.deployment_config.Pack(_LOCAL_DEPLOYMENT_CONFIG)
beam_dag_runner.BeamDagRunner._PIPELINE_NODE_DO_FN_CLS = _FakeComponentAsDoFn
beam_dag_runner.BeamDagRunner().run(self._pipeline)
self.assertEqual(
_component_executors, {
'my_example_gen':
text_format.Parse(
'class_path: "tfx.components.example_gen_executor"',
_PythonClassExecutableSpec()),
'my_transform':
text_format.Parse(
'class_path: "tfx.components.transform_executor"',
_PythonClassExecutableSpec()),
'my_trainer':
text_format.Parse(
'class_path: "tfx.components.trainer_executor"',
_PythonClassExecutableSpec()),
'my_importer':
None,
})
self.assertEqual(
_component_drivers, {
'my_example_gen':
text_format.Parse(
'class_path: "tfx.components.example_gen_driver"',
_PythonClassExecutableSpec()),
'my_transform':
None,
'my_trainer':
None,
'my_importer':
None,
})
# 'my_importer' has no upstream and can be executed in any order.
self.assertIn('my_importer', _executed_components)
_executed_components.remove('my_importer')
self.assertEqual(_executed_components,
['my_example_gen', 'my_transform', 'my_trainer'])
# Verifies that every component gets a not-None pipeline_run.
self.assertTrue(all(_conponent_to_pipeline_run.values()))
def testRun(self):
beam_dag_runner.BeamDagRunner().run(self._pipeline)
self.assertEqual(_executed_components,
['my_example_gen', 'my_transform', 'my_trainer'])
