def test_extract_independent_pipelines(self, df: DataFlowGraph):
settings.k8s.independent_graph.label = "pipeline"
df.add_streaming_app(self.get_k8s_app())
df.add_streaming_app(
self.get_k8s_app(
name="type2-app2",
input_topics="input-topic2",
error_topic="error-topic2",
output_topic="output-topic2",
pipeline="pipeline2",
)
)
df.extract_independent_pipelines()
assert "test-app" in df.independent_graphs
assert "pipeline2" in df.independent_graphs
df.graph.add_node("test-node")
df.extract_independent_pipelines()
assert "test-node" in df.independent_graphs
class StreamsExplorer:
context = settings.k8s.deployment.context
namespaces = settings.k8s.deployment.namespaces
def __init__(self, linking_service: LinkingService,
metric_provider: Type[MetricProvider]):
self.applications: Dict[str, K8sApp] = {}
self.kafka_connectors: List[KafkaConnector] = []
self.data_flow = DataFlowGraph(metric_provider=metric_provider)
self.linking_service = linking_service
def setup(self):
self.__setup_k8s_environment()
def update(self):
self.applications = {}
self.kafka_connectors = []
extractor_container.reset()
self.data_flow.reset()
self.__retrieve_deployments()
self.__retrieve_cron_jobs()
self.__get_connectors()
self.__create_graph()
def get_positioned_json_graph(self) -> dict:
return self.data_flow.get_positioned_graph()
def get_positioned_pipeline_json_graph(self, pipeline_name) -> dict:
return self.data_flow.get_positioned_pipeline_graph(pipeline_name)
def get_pipeline_names(self) -> List[str]:
return list(self.data_flow.independent_graphs.keys())
def get_metrics(self) -> List:
return self.data_flow.get_metrics()
def get_node_information(self, node_id: str):
node_type = self.data_flow.get_node_type(node_id)
if node_type == NodeTypesEnum.CONNECTOR:
config = KafkaConnect.get_connector_config(node_id)
return NodeInformation(
node_id=node_id,
node_type=node_type,
info=self.linking_service.connector_info +
get_displayed_information_connector(config),
)
if node_type == NodeTypesEnum.TOPIC or node_type == NodeTypesEnum.ERROR_TOPIC:
return NodeInformation(
node_id=node_id,
node_type=node_type,
info=self.linking_service.topic_info + [
NodeInfoListItem(
name="Schema",
value=SchemaRegistry.get_newest_topic_value_schema(
node_id),
type=NodeInfoType.JSON,
)
],
)
if node_type == NodeTypesEnum.STREAMING_APP:
info = get_displayed_information_deployment(
self.applications[node_id])
return NodeInformation(
node_id=node_id,
node_type=node_type,
info=self.linking_service.streaming_app_info + info,
)
if node_type in self.linking_service.sink_source_info:
return NodeInformation(
node_id=node_id,
node_type=NodeTypesEnum.SINK_SOURCE,
info=self.linking_service.sink_source_info[node_type],
)
def get_link(self, node_id: str, link_type: Optional[str]):
node_type = self.data_flow.get_node_type(node_id)
if node_type == NodeTypesEnum.CONNECTOR:
config = KafkaConnect.get_connector_config(node_id)
return self.linking_service.get_redirect_connector(
config, link_type)
if node_type == NodeTypesEnum.TOPIC or node_type == NodeTypesEnum.ERROR_TOPIC:
return self.linking_service.get_redirect_topic(node_id, link_type)
if node_type == NodeTypesEnum.STREAMING_APP:
return self.linking_service.get_redirect_streaming_app(
self.applications[node_id], link_type)
if node_type in self.linking_service.sink_source_redirects:
return self.linking_service.get_sink_source_redirects(
node_type, node_id)
def __setup_k8s_environment(self):
try:
if settings.k8s.deployment.cluster:
logger.info("Setup K8s environment in cluster")
kubernetes.config.load_incluster_config()
else:
logger.info("Setup K8s environment")
kubernetes.config.load_kube_config(context=self.context)
except kubernetes.config.ConfigException:
raise Exception("Could not load K8s environment configuration")
self.k8s_app_client = kubernetes.client.AppsV1Api()
self.k8s_batch_client = kubernetes.client.BatchV1beta1Api()
def __retrieve_deployments(self):
items = self.get_deployments() + self.get_stateful_sets()
for item in items:
try:
app = K8sApp.factory(item)
if app.is_streams_bootstrap_app():
self.applications[app.name] = app
except Exception as e:
logger.debug(e)
def get_deployments(self) -> List[V1Deployment]:
deployments: List[V1Deployment] = []
for namespace in self.namespaces:
logger.info(f"List deployments in namespace {namespace}")
deployments += self.k8s_app_client.list_namespaced_deployment(
namespace=namespace, watch=False).items
return deployments
def get_stateful_sets(self) -> List[V1StatefulSet]:
stateful_sets: List[V1StatefulSet] = []
for namespace in self.namespaces:
logger.info(f"List statefulsets in namespace {namespace}")
stateful_sets += self.k8s_app_client.list_namespaced_stateful_set(
namespace=namespace, watch=False).items
return stateful_sets
def __retrieve_cron_jobs(self):
logger.info("Retrieve cronjob descriptions")
cron_jobs = self.get_cron_jobs()
for cron_job in cron_jobs:
app: Optional[K8sApp] = extractor_container.on_cron_job(cron_job)
if app:
self.applications[app.name] = app
def get_cron_jobs(self) -> List[V1beta1CronJob]:
cron_jobs: List[V1beta1CronJob] = []
for namespace in self.namespaces:
logger.info(f"List cronjobs in namespace {namespace}")
cron_jobs += self.k8s_batch_client.list_namespaced_cron_job(
namespace=namespace, watch=False).items
return cron_jobs
def __get_connectors(self):
logger.info("Retrieve Kafka connectors")
self.kafka_connectors = KafkaConnect.connectors()
def __create_graph(self):
logger.info("Setup pipeline graph")
for _, app in self.applications.items():
self.data_flow.add_streaming_app(app)
for connector in self.kafka_connectors:
self.data_flow.add_connector(connector)
sources, sinks = extractor_container.get_sources_sinks()
for source in sources:
self.data_flow.add_source(source)
for sink in sinks:
self.data_flow.add_sink(sink)
# extract subgraphs
logger.info("Extract independent pipelines")
self.data_flow.extract_independent_pipelines()