def testCreateContainerComponentEnd2EndPipeline(self):
"""End-to-End test for container components."""
pipeline_name = 'kubeflow-container-e2e-test-{}'.format(
test_utils.random_id())
text_url = (
'https://storage.googleapis.com/ml-pipeline-playground/hamlet.txt')
pattern = 'art thou'
component_instances = download_grep_print_pipeline.create_pipeline_component_instances(
text_url=text_url,
pattern=pattern,
)
# Test that the pipeline can be executed successfully.
pipeline = self._create_pipeline(pipeline_name, component_instances)
self._compile_and_run_pipeline(pipeline=pipeline,
workflow_name=pipeline_name)
# Test if the correct value has been passed.
artifacts = self._get_artifacts_with_type_and_pipeline(
type_name='ExternalArtifact', pipeline_name=pipeline_name)
# There should be exactly two artifacts.
self.assertEqual(len(artifacts), 2)
for artifact in artifacts:
# TODO(b/150515270) Remove the '/data' suffix when b/150515270 is fixed.
artifact_value = tf.io.gfile.GFile(artifact.uri + '/data',
'r').read()
self.assertGreater(len(artifact_value), 100)
def _create_pipeline() -> pipeline_module.Pipeline:
"""Create sample container component pipeline."""
pipeline_name = _pipeline_name
pipeline_root = _pipeline_root
text_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/370cbcd/data/tinyshakespeare/input.txt'
pattern = 'art thou'
components = create_pipeline_component_instances(text_url, pattern)
# Use the default in-cluster MySql metadata config.
config = kubernetes_dag_runner.get_default_kubernetes_metadata_config()
return pipeline_module.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=components,
metadata_connection_config=config,
enable_cache=False,
)
