def test_pipeline_with_loop(self):
channel_one = channel.Channel(type_name='channel_one')
channel_two = channel.Channel(type_name='channel_two')
channel_three = channel.Channel(type_name='channel_three')
component_a = _FakeComponent('component_a', {})
component_b = _FakeComponent(name='component_b',
input_dict={
'a': component_a.outputs.output,
'one': channel_one
},
outputs=base_component.ComponentOutputs(
{'two': channel_two}))
component_c = _FakeComponent(name='component_b',
input_dict={
'a': component_a.outputs.output,
'two': channel_two
},
outputs=base_component.ComponentOutputs(
{'three': channel_three}))
component_d = _FakeComponent(name='component_b',
input_dict={
'a': component_a.outputs.output,
'three': channel_three
},
outputs=base_component.ComponentOutputs(
{'one': channel_one}))
with self.assertRaises(RuntimeError):
pipeline.Pipeline(pipeline_name='a',
pipeline_root='b',
log_root='c',
components=[
component_c, component_d, component_b,
component_a
])
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for SlackComponent.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
slack_blessing_output = [types.TfxType('ModelBlessingPath')]
return base_component.ComponentOutputs({
'slack_blessing':
channel.Channel(type_name='ModelBlessingPath',
static_artifact_collection=slack_blessing_output),
})
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for Evaluator.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
output_artifact_collection = [types.TfxType('ModelEvalPath')]
return base_component.ComponentOutputs({
'output':
channel.Channel(
type_name='ModelEvalPath',
static_artifact_collection=output_artifact_collection),
})
def _create_outputs(self):
"""Creates outputs for ExampleValidator.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
output_artifact_collection = [types.TfxType('SchemaPath')]
return base_component.ComponentOutputs({
'output':
channel.Channel(
type_name='SchemaPath',
static_artifact_collection=output_artifact_collection)
})
def _create_outputs(self):
"""Creates outputs for Pusher.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
model_push_artifact_collection = [types.TfxType('ModelPushPath',)]
return base_component.ComponentOutputs({
'model_push':
channel.Channel(
type_name='ModelPushPath',
static_artifact_collection=model_push_artifact_collection),
})
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for ModelValidator.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
blessing_artifact_collection = [types.TfxArtifact('ModelBlessingPath')]
return base_component.ComponentOutputs({
'blessing':
channel.Channel(
type_name='ModelBlessingPath',
static_artifact_collection=blessing_artifact_collection),
})
def _create_outputs(self):
"""Creates outputs for CsvExampleGen.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
output_artifact_collection = [
types.TfxType('ExamplesPath', split=split)
for split in types.DEFAULT_EXAMPLE_SPLITS
]
return base_component.ComponentOutputs({
'examples':
channel.Channel(
type_name='ExamplesPath',
static_artifact_collection=output_artifact_collection)
})
def _create_outputs(self):
"""Creates outputs for BigQueryExampleGen.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
output_artifact_collection = [
types.TfxType('ExamplesPath', split=split.name)
for split in self._output_config.split_config.splits
]
return base_component.ComponentOutputs({
'examples':
channel.Channel(
type_name='ExamplesPath',
static_artifact_collection=output_artifact_collection)
})
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for ExampleGen.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
output_artifact_collection = [
types.TfxArtifact('ExamplesPath', split=split_name)
for split_name in utils.generate_output_split_names(
self._input_config, self._output_config)
]
return base_component.ComponentOutputs({
'examples':
channel.Channel(
type_name='ExamplesPath',
static_artifact_collection=output_artifact_collection)
})
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for StatisticsGen.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
# pylint: disable=g-complex-comprehension
output_artifact_collection = [
types.TfxType(
'ExampleStatisticsPath',
split=split,
) for split in types.DEFAULT_EXAMPLE_SPLITS
]
# pylint: enable=g-complex-comprehension
return base_component.ComponentOutputs({
'output':
channel.Channel(
type_name='ExampleStatisticsPath',
static_artifact_collection=output_artifact_collection)
})
def _create_outputs(self) -> base_component.ComponentOutputs:
"""Creates outputs for Transform.
Returns:
ComponentOutputs object containing the dict of [Text -> Channel]
"""
transform_output_artifact_collection = [types.TfxArtifact('TransformPath',)]
transformed_examples_artifact_collection = [
types.TfxArtifact('ExamplesPath', split=split)
for split in types.DEFAULT_EXAMPLE_SPLITS
]
return base_component.ComponentOutputs({
'transform_output':
channel.Channel(
type_name='TransformPath',
static_artifact_collection=transform_output_artifact_collection
),
'transformed_examples':
channel.Channel(
type_name='ExamplesPath',
static_artifact_collection=transformed_examples_artifact_collection
),
})
def _create_outputs(self) -> base_component.ComponentOutputs:
return base_component.ComponentOutputs({
'output': channel.Channel(type_name=self.component_name),
})
