def test_pipeline_graph(self, df: DataFlowGraph): df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( pipeline="pipeline1", ) ) ) df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app2", input_topics="input-topic2", error_topic="error-topic2", output_topic="output-topic2", pipeline="pipeline2", multiple_inputs="0=output-topic", ) ) ) assert len(df.pipelines) == 2 assert "pipeline1" in df.pipelines assert "pipeline2" in df.pipelines pipeline1 = df.pipelines["pipeline1"] pipeline2 = df.pipelines["pipeline2"] assert set(pipeline1.nodes) == { "input-topic", "output-topic", "test-app", "error-topic", } assert set(pipeline2.nodes) == { "output-topic", "input-topic2", "output-topic2", "test-app2", "error-topic2", } df.add_sink(Sink("test-sink", "output-topic")) assert "test-sink" in pipeline1.nodes source_connector = KafkaConnector( name="test-source-connector", type=KafkaConnectorTypesEnum.SOURCE, topics=["input-topic2", "source-topic"], config={}, ) df.add_connector(source_connector) assert "test-source-connector" in pipeline2.nodes assert "source-topic" in pipeline2.nodes
def mock_get_deployments(*_): return [ get_streaming_app_deployment("streaming-app1", "input-topic1", "output-topic1", "error-topic1"), get_streaming_app_deployment("streaming-app2", "input-topic2", "output-topic2", "error-topic2"), get_streaming_app_deployment( "streaming-app3", "input-topic3", "output-topic3", "error-topic3", pipeline="pipeline2", ), ]
def mock_get_deployments(*args, **kwargs): return [ get_streaming_app_deployment( "streaming-app1", "input-topic1", "output-topic1", "error-topic1", ), get_streaming_app_deployment( "streaming-app2", "input-topic2", "output-topic2", "error-topic2", ), ]
def mock_get_deployments_after(*_): return [ get_streaming_app_deployment( "streaming-app1", "input-topic1", "output-topic1", "error-topic1", ), get_streaming_app_deployment( "streaming-app2", "input-topic2", "output-topic2", "error-topic2", ), ]
def test_pipeline_cronjob(self, df: DataFlowGraph): df.add_streaming_app( K8sApp.factory( get_streaming_app_cronjob( error_topic=None, pipeline="pipeline1", ) ) ) df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( input_topics="output-topic", error_topic=None, output_topic="output-topic2", pipeline="pipeline1", ) ) ) assert len(df.pipelines) == 1 assert "pipeline1" in df.pipelines pipeline1 = df.pipelines["pipeline1"] assert set(pipeline1.nodes) == { "test-cronjob", "output-topic", "test-app", "output-topic2", }
def test_get_displayed_information_deyployment(self, monkeypatch): monkeypatch.setattr( settings.k8s, "displayed_information", [ { "name": "Labels", "key": "metadata.labels" }, { "name": "Pipeline", "key": "metadata.labels.pipeline" }, ], ) k8s_app = get_streaming_app_deployment( "streaming-app1", "input-topic1", "output-topic1", "error-topic1", pipeline="pipeline1", ) output = get_displayed_information_deployment( K8sAppDeployment(k8s_app)) assert len(output) == 2 value = output[0].value assert "pipeline" in value assert output[1] == NodeInfoListItem(name="Pipeline", value="pipeline1", type=NodeInfoType.BASIC)
def test_init(self): k8s_objects = [ get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", ), get_streaming_app_stateful_set( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", ), ] k8s_apps = [K8sApp.factory(k8s_object) for k8s_object in k8s_objects] for k8s_app in k8s_apps: assert k8s_app.name == "test-app" assert k8s_app.error_topic == "error-topic" assert k8s_app.output_topic == "output-topic" assert k8s_app.input_topics == ["input-topic"] assert isinstance(k8s_apps[0], K8sAppDeployment) assert isinstance(k8s_apps[1], K8sAppStatefulSet) assert k8s_apps[1].get_service_name() == "test-service"
async def test_get_positioned_json_graph(self, df: DataFlowGraph): df.add_streaming_app(K8sApp.factory(get_streaming_app_deployment())) await df.get_positioned_graph() nodes = df.graph.nodes(data=True) for _, data in iter(nodes): assert data.get("x") is not None assert data.get("y") is not None
def test_is_streams_bootstrap_app(self): streams_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic=None, )) assert streams_app.is_streams_bootstrap_app() non_streams_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics=None, output_topic=None, error_topic=None, )) assert not non_streams_app.is_streams_bootstrap_app()
def test_multiple_inputs(self): k8s_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic1,input-topics2", output_topic="output-topic", error_topic="error-topic", )) assert k8s_app.input_topics == ["input-topic1", "input-topics2"]
def deployments(self): return [ get_streaming_app_deployment("streaming-app1", "input-topic1", "output-topic1", "error-topic1"), get_streaming_app_deployment( "streaming-app2", "input-topic2", "output-topic2", "error-topic2", consumer_group="consumer-group2", ), get_streaming_app_deployment( "streaming-app3", "input-topic3", "output-topic3", "error-topic3", pipeline="pipeline2", ), ]
def test_node_attributes(self, df: DataFlowGraph): df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app1", pipeline="pipeline1", ) ) ) assert df.graph.nodes["test-app1"].get(ATTR_PIPELINE) == "pipeline1" df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app2", pipeline=None, ) ) ) assert df.graph.nodes["test-app2"].get(ATTR_PIPELINE) is None
def test_extra_input_topics(self): k8s_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", multiple_inputs="0=test1,1=test2", env_prefix="TEST_", )) assert k8s_app.extra_input_topics == ["test1", "test2"]
def test_add_source(self, df: DataFlowGraph): source = Source( name="test-source", node_type="test-type", target="test-app", ) df.add_streaming_app(K8sApp.factory(get_streaming_app_deployment())) df.add_source(source) assert len(df.graph.nodes) == 5 assert df.graph.has_edge("test-source", "test-app") assert len(df.pipelines) == 0
def test_error_topic_undefined(self): k8s_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic=None, )) assert k8s_app.name == "test-app" assert k8s_app.error_topic is None assert k8s_app.output_topic == "output-topic" assert k8s_app.input_topics == ["input-topic"]
def test_env_prefix_support(self): k8s_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", env_prefix="TEST_", )) assert k8s_app.name == "test-app" assert k8s_app.error_topic == "error-topic" assert k8s_app.output_topic == "output-topic" assert k8s_app.input_topics == ["input-topic"]
def test_multiple_pipelines_apps(self, df: DataFlowGraph): """Ensures apps have separate pipelines despite them being connected.""" df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app1", input_topics="input-topic1", error_topic=None, output_topic="output-topic1", pipeline="pipeline1", ) ) ) df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app2", input_topics="output-topic1", error_topic=None, output_topic="output-topic2", pipeline="pipeline2", ) ) ) assert len(df.pipelines) == 2 assert "pipeline1" in df.pipelines assert "pipeline2" in df.pipelines pipeline1 = df.pipelines["pipeline1"] pipeline2 = df.pipelines["pipeline2"] assert set(pipeline1.nodes) == { "test-app1", "input-topic1", "output-topic1", } assert set(pipeline2.nodes) == { "test-app2", "output-topic1", "output-topic2", }
def test_attributes(self): k8s_app = K8sAppDeployment( get_streaming_app_deployment( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", multiple_outputs="0=test1,1=test2", env_prefix="TEST_", pipeline="pipeline1", )) assert k8s_app.attributes["pipeline"] == "pipeline1" assert len(k8s_app.attributes) == 1
def test_add_streaming_app(self, df: DataFlowGraph): df.add_streaming_app(K8sApp.factory(get_streaming_app_deployment())) assert len(df.graph.nodes) == 4 assert df.graph.has_edge("input-topic", "test-app") assert df.graph.has_edge("test-app", "output-topic") assert df.graph.has_edge("test-app", "error-topic") # should have multiple input topic df.reset() df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment(input_topics="input-topic1,input-topic2") ) ) assert len(df.graph.nodes) == 5 assert df.graph.has_edge("input-topic1", "test-app") assert df.graph.has_edge("input-topic2", "test-app") assert df.graph.has_edge("test-app", "output-topic") assert df.graph.has_edge("test-app", "error-topic") df.reset() df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( multiple_outputs="1=extra-output1,2=extra-output2" ) ) ) assert len(df.graph.nodes) == 6 assert df.graph.has_edge("input-topic", "test-app") assert df.graph.has_edge("test-app", "output-topic") assert df.graph.has_edge("test-app", "error-topic") assert df.graph.has_edge("test-app", "extra-output1") assert df.graph.has_edge("test-app", "extra-output2")
def get_k8s_app( name="test-app", input_topics="input-topic", output_topic="output-topic", error_topic="error-topic", multiple_inputs=None, multiple_outputs=None, pipeline=None, ): return K8sAppDeployment( get_streaming_app_deployment( name, input_topics, output_topic, error_topic, multiple_inputs, multiple_outputs, pipeline=pipeline, ) )
def test_add_connector(self, df: DataFlowGraph): sink_connector = KafkaConnector( name="test-sink-connector", type=KafkaConnectorTypesEnum.SINK, topics=["output-topic"], config={}, error_topic="dead-letter-topic", ) source_connector = KafkaConnector( name="test-source-connector", type=KafkaConnectorTypesEnum.SOURCE, topics=["input-topic", "input-topic2"], config={}, ) df.add_streaming_app(K8sApp.factory(get_streaming_app_deployment())) df.add_connector(sink_connector) df.add_connector(source_connector) assert len(df.graph.nodes) == 8 assert df.graph.has_edge("output-topic", "test-sink-connector") assert df.graph.has_edge("test-sink-connector", "dead-letter-topic") assert df.graph.has_edge("test-source-connector", "input-topic") assert df.graph.has_edge("test-source-connector", "input-topic2") assert len(df.pipelines) == 0
def test_multiple_pipelines_sink_source(self, df: DataFlowGraph): df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app1", input_topics="input-topic1", error_topic="error-topic1", output_topic="output-topic1", pipeline="pipeline1", ) ) ) df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app2", input_topics="input-topic2", error_topic="error-topic2", output_topic="output-topic2", pipeline="pipeline2", ) ) ) assert len(df.pipelines) == 2 assert "pipeline1" in df.pipelines assert "pipeline2" in df.pipelines pipeline1 = df.pipelines["pipeline1"] pipeline2 = df.pipelines["pipeline2"] assert set(pipeline1.nodes) == { "test-app1", "input-topic1", "output-topic1", "error-topic1", } assert set(pipeline2.nodes) == { "test-app2", "input-topic2", "output-topic2", "error-topic2", } sink_connector = KafkaConnector( name="test-sink-connector", type=KafkaConnectorTypesEnum.SINK, topics=["output-topic1", "output-topic2"], config={}, ) df.add_connector(sink_connector) assert "test-sink-connector" in df.graph.nodes assert "test-sink-connector" in pipeline1.nodes assert "test-sink-connector" in pipeline2.nodes df.add_sink(Sink("test-sink", "test-sink-connector")) assert "test-sink" in df.graph.nodes assert "test-sink" in pipeline1.nodes assert "test-sink" in pipeline2.nodes source_connector = KafkaConnector( name="test-source-connector", type=KafkaConnectorTypesEnum.SOURCE, topics=["input-topic1", "input-topic2"], config={}, ) df.add_connector(source_connector) assert "test-source-connector" in df.graph.nodes assert "test-source-connector" in pipeline1.nodes assert "test-source-connector" in pipeline2.nodes df.add_source(Source("test-source", "test-source-connector")) assert "test-source" in df.graph.nodes assert "test-source" in pipeline1.nodes assert "test-source" in pipeline2.nodes unrelated_sink_connector = KafkaConnector( name="unrelated-sink-connector", type=KafkaConnectorTypesEnum.SINK, topics=["input-topic1"], config={}, ) df.add_connector(unrelated_sink_connector) assert "unrelated-sink-connector" in df.graph.nodes assert "unrelated-sink-connector" not in pipeline1.nodes assert "unrelated-sink-connector" not in pipeline2.nodes unrelated_source_connector = KafkaConnector( name="unrelated-source-connector", type=KafkaConnectorTypesEnum.SOURCE, topics=["output-topic1"], config={}, ) df.add_connector(unrelated_source_connector) assert "unrelated-source-connector" in df.graph.nodes assert "unrelated-source-connector" not in pipeline1.nodes assert "unrelated-source-connector" not in pipeline2.nodes
def test_verify_connector_exists_in_pipeline(self, df: DataFlowGraph): """Verify that connector exists in specific pipeline before adding it from a sink or source.""" df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app1", input_topics="input-topic1", error_topic=None, output_topic="output-topic1", pipeline="pipeline1", ) ) ) df.add_streaming_app( K8sApp.factory( get_streaming_app_deployment( name="test-app2", input_topics="input-topic2", error_topic=None, output_topic="output-topic2", pipeline="pipeline2", ) ) ) sink_connector1 = KafkaConnector( name="sink-connector1", type=KafkaConnectorTypesEnum.SINK, topics=["output-topic1"], config={}, ) df.add_connector(sink_connector1) sink_connector2 = KafkaConnector( name="sink-connector2", type=KafkaConnectorTypesEnum.SINK, topics=["output-topic2"], config={}, ) df.add_connector(sink_connector2) sink = Sink( name="test-sink", node_type="test-type", source="sink-connector1", ) df.add_sink(sink) sink.source = "sink-connector2" df.add_sink(sink) assert len(df.pipelines) == 2 assert "pipeline1" in df.pipelines assert "pipeline2" in df.pipelines pipeline1 = df.pipelines["pipeline1"] pipeline2 = df.pipelines["pipeline2"] assert "sink-connector1" in pipeline1.nodes assert "sink-connector1" not in pipeline2.nodes assert "sink-connector2" in pipeline2.nodes assert "sink-connector2" not in pipeline1.nodes assert "test-sink" in pipeline1.nodes assert "test-sink" in pipeline2.nodes assert set(pipeline1.nodes) == { "test-app1", "input-topic1", "output-topic1", "sink-connector1", "test-sink", } assert set(pipeline2.nodes) == { "test-app2", "input-topic2", "output-topic2", "sink-connector2", "test-sink", }
def test_get_node_type(self, df: DataFlowGraph): df.add_streaming_app(K8sApp.factory(get_streaming_app_deployment())) assert df.get_node_type("test-app") == "streaming-app"