def testCreateContainerComponentEnd2EndPipeline(self): """End-to-End test for container components.""" pipeline_name = 'kubeflow-container-e2e-test-{}'.format( test_utils.random_id()) text_url = ( 'https://storage.googleapis.com/ml-pipeline-playground/hamlet.txt') pattern = 'art thou' component_instances = download_grep_print_pipeline.create_pipeline_component_instances( text_url=text_url, pattern=pattern, ) # Test that the pipeline can be executed successfully. pipeline = self._create_pipeline(pipeline_name, component_instances) self._compile_and_run_pipeline(pipeline=pipeline, workflow_name=pipeline_name) # Test if the correct value has been passed. artifacts = self._get_artifacts_with_type_and_pipeline( type_name='ExternalArtifact', pipeline_name=pipeline_name) # There should be exactly two artifacts. self.assertEqual(len(artifacts), 2) for artifact in artifacts: # TODO(b/150515270) Remove the '/data' suffix when b/150515270 is fixed. artifact_value = tf.io.gfile.GFile(artifact.uri + '/data', 'r').read() self.assertGreater(len(artifact_value), 100)
def _create_pipeline() -> pipeline_module.Pipeline: """Create sample container component pipeline.""" pipeline_name = _pipeline_name pipeline_root = _pipeline_root text_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/370cbcd/data/tinyshakespeare/input.txt' pattern = 'art thou' components = create_pipeline_component_instances(text_url, pattern) # Use the default in-cluster MySql metadata config. config = kubernetes_dag_runner.get_default_kubernetes_metadata_config() return pipeline_module.Pipeline( pipeline_name=pipeline_name, pipeline_root=pipeline_root, components=components, metadata_connection_config=config, enable_cache=False, )