def cb(c):
with Diagram('Context Broker Pod',
filename='cb',
show=False,
graph_attr={'pad': '0.0'}):
_kafka = Kafka('Kafka')
_zk = Zookeeper('Zookeeper')
_logstash = Logstash('Logstash')
_elasticsearch = Elasticsearch('Elasticsearch')
_kibana = Kibana('Kibana')
_cb_man = Python('cb-manager')
with Cluster('elasticsearch-config'):
_elasticsearch_cfg = [
CM('elasticsearch.yml'),
CM('log4j2.properties')
]
_ = _elasticsearch_cfg - _elasticsearch
with Cluster('logstash-config'):
_logstash_cfg = [CM('logstash.yml'), CM('log4j2.properties')]
_ = _logstash_cfg - _logstash
with Cluster('logstash-pipeline'):
_logstash_pipe = [CM('data.conf')]
_ = _logstash_pipe - _logstash
_zk - _kafka >> _logstash >> _elasticsearch << _cb_man
_elasticsearch << _kibana
_logstash << _cb_man >> _kafka
def process_media(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates media processing diagrams """
output = f"{output_path}/media_processing"
with Diagram(
"Media Processing",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
cli = Python("CLI")
with Cluster("Serverless"):
source = SQS("tasks queue")
with Cluster("Concurrent Processing"):
handlers = Lambda("convert image\nencode video")
with Cluster("Lambda layers"):
[Layer("ImageMagic"), Layer("ffmpeg")]
src = S3("input\nmedia files")
dest = S3("output\nmedia files")
cli >> source >> handlers << src
handlers >> dest
return f"{output}.{output_format}"
def cb(c):
with Diagram('cb', show=False):
_kafka = Kafka('Kafka')
_zk = Zookeeper('Zookeeper')
_logstash = Logstash('Logstash')
_elasticsearch = Elasticsearch('Elasticsearch')
_cb_man = Python('cb-manager')
with Cluster('elasticsearch-config'):
_elasticsearch_cfg = [
CM('elasticsearch.yml'),
CM('log4j2.properties')
]
_ = _elasticsearch_cfg - _elasticsearch
with Cluster('logstash-config'):
_logstash_cfg = [CM('logstash.yml'), CM('log4j2.properties')]
_ = _logstash_cfg - _logstash
with Cluster('logstash-pipeline'):
_logstash_pipe = [CM('data.conf')]
_ = _logstash_pipe - _logstash
_zk - _kafka >> _logstash >> _elasticsearch << _cb_man
_logstash << _cb_man >> _kafka
def cli(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates CLI diagrams """
output = f"{output_path}/cli"
with Diagram(
"CLI",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
cli = Python("CLI")
with Cluster("targets"):
with Cluster("local resources"):
data = Storage("build objects\nassets")
layers = Layer("build\nLambda layers")
server = Server("media\nlocal build")
with Cluster("cloud resources"):
db = Dynamodb("create DB\nseed DB")
s3 = S3("sync data")
lbd = Lambda("resize images\nencode video")
sqs = SQS("queue\nmedia generate\ntasks")
cli >> data >> cli
cli >> s3
cli >> db
cli >> lbd
cli >> sqs
cli >> layers
cli >> server
return f"{output}.{output_format}"
def generate_overview_diagram():
graph_attr = {
"bgcolor": "transparent"
}
with Diagram("Overview", show=False, filename="bin/overview", graph_attr=graph_attr):
with Cluster("Client"):
webapp = Angular("webapp")
with Cluster("API Services"):
status_service = Python("status_service")
task_service = Python("task_service")
worker_service = Python("worker_service")
metrics_service = Python("metrics_service")
with Cluster("Intermediate Services"):
with Cluster("Clearly Client Cluster"):
clearly_client = Python("clearly_client")
with Cluster("Backend Services"):
with Cluster("Clearly Server Cluster"):
clearly_server = Python("clearly_server")
with Cluster("External Connections"):
with Cluster("Message Broker"):
redis = Redis("redis")
with Cluster("Monitoring"):
grafana = Grafana("grafana")
prometheus = Prometheus("prometheus")
webapp >> status_service << clearly_client
webapp >> task_service << clearly_client
webapp >> worker_service << clearly_client
clearly_client >> clearly_server >> redis
metrics_service << prometheus
metrics_service >> clearly_server
from diagrams import Cluster, Diagram
from diagrams.generic.database import SQL
from diagrams.aws.database import RDSMysqlInstance
from diagrams.generic.device import Mobile
from diagrams.programming.language import Python
from diagrams.custom import Custom
from diagrams.onprem.monitoring import Grafana
with Diagram("Event processing",show=False):
phone = Mobile("Phone running Owntracks")
with Cluster("Services"):
connector = Python("Connector")
mysql = RDSMysqlInstance("MYSQL DB")
mqtt = Custom("MQTT broker", "./mosquitto.png")
grafana = Grafana("Grafana")
phone >> mqtt >> connector >> mysql << grafana
platform_io_logo_path = os.path.join(os.getcwd(), "resources",
"platform_io.png")
esp32_path = os.path.join(os.getcwd(), "resources", "esp32.png")
arduino_nano_path = os.path.join(os.getcwd(), "resources", "arduino_nano.png")
arduino_mega_pro_path = os.path.join(os.getcwd(), "resources",
"arduino_mega_pro.png")
with Diagram(name="One-Green Deploy Center",
filename="export/deploy_center",
direction="LR",
show=False):
streamlit = Custom("Webapp", streamlit_logo_path)
with Cluster("Ansible deployment"):
ansible = Custom("Ansible", ansible_logo_path)
iot_edge_agent = Python("Edge IoT Agent\n Deployment")
opi_zero = Custom("\nOrange Pi Zero LTS", opi_zero_path)
streamlit >> ansible >> iot_edge_agent >> opi_zero
kube_setup = Custom("Kubernetes\nSetup", kubernetes_logo_path)
helm_deploy = Custom("Helm\nDeployment", helm_logo_path)
raspi_4 = Custom("Raspberry 4", raspi_4_path)
streamlit >> ansible >> [kube_setup, helm_deploy] >> raspi_4
with Cluster("Platform deployment"):
platform_io = Custom("PlatfromIO Core \nbuild and flash firmware",
platform_io_logo_path)
esp32 = Custom("Esp32\nplatform", esp32_path)
arduino_nano = Custom("Arduino Nano\nplatform", arduino_nano_path)
arduino_mega_pro = Custom("Arduino Mega pro\nplatform ",
arduino_mega_pro_path)
from diagrams import Cluster, Diagram, Edge
from diagrams.onprem.iac import Ansible
from diagrams.onprem.monitoring import Grafana, Prometheus
from diagrams.onprem.queue import RabbitMQ
from diagrams.programming.framework import Django
from diagrams.programming.language import Python
with Diagram("MQ Cluster", show=False, direction="TB"):
provisioner = Ansible("mgmt-core")
metrics = Grafana("monitoring") >> Edge(color="firebrick", style="dashed") >> Prometheus("rabbitmq_exporter")
producer = Django("Django App")
consumer = Python("Celery worker")
with Cluster("RabbitMQ 3-Node Cluster"):
mq_cluster = [RabbitMQ("leader"),
RabbitMQ("follower_1"),
RabbitMQ("follower_2")]
provisioner >> mq_cluster
producer >> Edge(color="brown") >> mq_cluster << Edge(color="brown") << consumer
mq_cluster << metrics
with Diagram("MQ Single", show=False, direction="TB"):
provisioner = Ansible("mgmt-core")
metrics = Grafana("monitoring") >> Edge(color="firebrick", style="dashed") >> Prometheus("rabbitmq_exporter")
producer = Django("Django App")
from diagrams import Cluster, Diagram
from diagrams.azure.database import SQLDatabases
from diagrams.generic.database import SQL
from diagrams.programming.language import Python, R
with Diagram("Data Quality Model", show=False, outformat="jpg"):
dm = SQLDatabases("Data Model")
sample = SQL("Sample")
metadata = SQLDatabases("Metadata")
QA = SQL("Quality Assessment")
with Cluster("Source"):
source = [metadata, dm, sample]
dm >> metadata
dm >> sample
with Cluster("Sample Analysis"):
SA = sample >> Python("Sample metrics") >> R("Statistical analysis")
with Cluster("Analysis"):
dm >> Python("Analysis") >> QA
metadata >> Python("Analysis") >> QA
with Cluster("Quality assessment"):
SA >> QA
QA >> SQL("Quality report")
with Cluster(" Envoie_de_la notification "):
flux_2 = Eventbridge(
"planifie l'execution \nde la fonction\nscan_user du lundi \nau vendredi à 8H00"
) >> Lambda("Handler:lambdascan_handeler")
#with Cluster("Enregistrement_de_l'utilisateur "):
flux_1 = Route53(
"redirige le trafic \n de mtchou-mouh.mongulu.cm \n vers le bucket S3 de meme nom"
) >> S3(
"3 PAGES WEB :\n-index.html\n-demo.html\n-error.html"
) >> APIGateway(
"-OPTIONS:résout\nproblèmeCORS\n-POST:donne les\n informations à LAMBDA"
) >> Lambda(
"Handler:lambda.register_handler\n-Enregistrement information dans\n DynamoDB table Register\n-Envoie mail de verification \nvia AmazoneSES"
)
with Cluster("Programming"):
languages = [Python("BackEnd"), Javascript("FrontEnd")]
with Cluster("BackEnd_1"):
Backend_1 = [DDB("DataBase"), SES("verification")]
with Cluster("BackEnd_2"):
Backend_2 = [
S3("storage"),
SES("notifier"),
DDB("DataBase"),
Textract("Extraction")
]
languages >> flux_1 >> Backend_1
languages >> flux_2 >> Backend_2
def main():
graph_attr = {
"fontsize": "45",
'overlap_scaling': '100',
'size': '24!',
'ratio': 'expand'
}
with Diagram(name='Automation Framework Compose',
direction='LR',
graph_attr=graph_attr):
with Cluster('Docker Cluster'):
docker = Docker('Docker')
with Cluster('container1'):
python_container = Python('APIs\nOther Microservices')
with Cluster('Docker Registry'):
docker_registry_container = Docker('Docker Registry\ntcp:5000')
with Cluster('Docker Registry Browser'):
docker_registry_browser_container = Python(
'Docker Registry Browser\ntcp:8088')
with Cluster('BatFish'):
batfish_container = Custom(
'BatFish\ntcp:8888\ntcp:9997\ntcp:9996',
'custom_icons/BatFish.png')
with Cluster('Kafka Cluster'):
with Cluster('Zookeeper'):
Zookeeper('Zookeeper\ntcp:2181')
with Cluster('REST Proxy'):
rest_proxy = Custom('REST Proxy\ntcp:8082',
'custom_icons/REST-API.png')
with Cluster('Control Center'):
control_center = Kafka('Control Center\ntcp:9021')
with Cluster('Schema Registry'):
schema_registry = Storage('Schema Registry\ntcp:8081')
with Cluster('Brokers'):
broker_1 = Kafka('Broker 1\ntcp:9092')
kafka_brokers = [
broker_1,
Kafka('Broker 2\ntcp:9093'),
Kafka('Broker 3\ntcp:9094')
]
with Cluster('Secrets Managers'):
vault = Vault('HashiCorp Vault\ntcp:8200')
secrets_managers = [
vault,
]
with Cluster('Logging and Search'):
with Cluster('ELK Stack'):
elastic_search = Elasticsearch('Elastic Search\ntcp:9200')
kibana = Kibana('Kibana\ntcp:5601')
logstash = Logstash('Logstash\ntcp:5044')
search_log = [elastic_search, kibana, logstash]
with Cluster('Influxdb'):
infulxdb = Influxdb('Influxdb\ntcp:8086')
with Cluster('Grafana'):
grafana = Grafana('Grafana\ntcp:3000')
with Cluster('Inventory and Connectivity'):
with Cluster('Inventory'):
nautobot = Custom('Nautobot\ntcp:8000',
'custom_icons/Nautobot.jpeg')
with Cluster('Database'):
with Cluster('Mongo dB'):
mongodb = Mongodb('MongoDb\ntcp:27017')
mongodb_express = Mongodb('MongoDb Express\ntcp:8181')
mongo_group = [mongodb, mongodb_express]
with Cluster('CI/CD'):
team_city = TC('TeamCity')
kafka_brokers - python_container
python_container - vault
python_container - nautobot
nautobot - logstash
python_container - logstash
nautobot - infulxdb
python_container - infulxdb
python_container - mongodb
def main():
graph_attr = {
"fontsize": "45",
'overlap_scaling': '100',
'size': '24!',
'ratio': 'expand'
}
with Diagram(name='Automation Framework Swarm', direction='LR', graph_attr=graph_attr):
with Cluster('Docker Cluster'):
docker = Docker('Docker')
with Cluster('container1'):
python_container = Python('APIs\nOther Microservices')
with Cluster('Kafka Cluster'):
with Cluster('Zookeeper'):
Zookeeper('Zookeeper\ntcp:2181')
with Cluster('REST Proxy'):
rest_proxy = Custom('REST Proxy\ntcp:8082', 'custom_icons/REST-API.png')
with Cluster('Control Center'):
control_center = Kafka('Control Center\ntcp:9021')
with Cluster('Schema Registry'):
schema_registry = Storage('Schema Registry\ntcp:8081')
with Cluster('Brokers'):
broker_1 = Kafka('Broker 1\ntcp:9092')
kafka_brokers = [
broker_1,
Kafka('Broker 2\ntcp:9093'),
Kafka('Broker 3\ntcp:9094')
]
with Cluster('Secrets Managers'):
vault = Vault('HashiCorp Vault\ntcp:8200')
secrets_managers = [
vault,
]
with Cluster('Logging and Search'):
with Cluster('Search and Logging'):
elastic_search = Elasticsearch('Elastic Search\ntcp:9200')
kibana = Kibana('Kibana\ntcp:5601')
logstash = Logstash('Logstash\ntcp:5044')
search_log = [
elastic_search,
kibana,
logstash
]
with Cluster('Inventory and Connectivity'):
with Cluster('Inventory'):
nautobot = Custom('Nautobot\ntcp:8000', 'custom_icons/Nautobot.jpeg')
kafka_brokers - python_container
python_container - vault
python_container - nautobot
nautobot - logstash
python_container - logstash
from diagrams import Cluster, Diagram, Edge
from diagrams.oci.governance import Logging
from source.util.architecture.custom import Finansportalen, Finn, Posten, SIFO, Sqlite, SSB, \
Skatteetaten
from source.util.architecture.settings import name, font_size
with Diagram(name=name,
show=False,
direction="TB",
outformat="pdf",
node_attr={"fontsize": font_size}):
users = User()
gui = Client()
bash = Bash("")
main = Python("main")
with Cluster("presentation layer (source/ui)",
graph_attr={"fontsize": font_size}):
with Cluster("package: views", graph_attr={"fontsize": font_size}):
views = Python("views")
with Cluster("forms", graph_attr={"fontsize": font_size}):
forms = FileStorage("forms")
with Cluster("logs", graph_attr={"fontsize": font_size}):
ui_log = Logging("ui")
with Cluster("package: models", graph_attr={"fontsize": font_size}):
models = Python("models")
with Cluster("package: util", graph_attr={"fontsize": font_size}):
from diagrams import Diagram, Cluster
from diagrams.programming.framework import Vue
from diagrams.programming.language import Go
from diagrams.programming.language import Python
with Diagram("Diagrams Sandbox", show=False):
with Cluster("gVisor"):
diagrams = Python("diagrams")
Vue("diagrams-front") >> Go("diagrams-backend") >> diagrams
from diagrams import Diagram, Cluster
from diagrams.gcp.compute import GCF
from diagrams.gcp.storage import GCS
from diagrams.onprem.iac import Terraform
from diagrams.programming.language import Python
Terraform._height = 0.9
Python._height = 0.9
diagram_kwargs = dict(direction="LR", filename="docs/diagram", show=False)
with Diagram("GCF Data Mining Example", **diagram_kwargs):
with Cluster("DevOps & Source Code") as devops:
source_code = GCS("Source Code Bucket")
python = Python()
state = GCS("Terraform State Bucket")
terraform = Terraform()
with Cluster("Application") as app:
function = GCF("Data Mining Service")
data = GCS("Data Bucket")
source_code >> python >> function
state >> terraform >> function
function >> data
from diagrams.onprem.database import PostgreSQL
graph_attr = {"bgcolor": "transparent"}
with Diagram("Basic Architecture", outformat="png", graph_attr=graph_attr):
with Cluster("Docker Swarm"):
host = [Server("Cloud Host")]
with Cluster("Frontend"):
frontend = Vue("Nuxt + Vuetify")
host << frontend
with Cluster("Search Engine"):
search = Rust("MeiliSearch")
frontend << search
with Cluster("API"):
api = Python("FastAPI")
search >> api
api >> search
with Cluster("Database"):
db = PostgreSQL("Recipes")
api >> db
db >> api
with Cluster("Web Scraper"):
scraper = Python("Scrapy")
scraper >> api
from diagrams.onprem.client import Users, Client
from diagrams.k8s.compute import Pod
from diagrams.k8s.infra import Master
from diagrams.k8s.controlplane import API
from diagrams.k8s.network import Ingress, Service
from diagrams.onprem.network import Traefik
from diagrams.programming.language import Python
with Diagram("Simplified Orchestra Architecture Diagram", show=False):
dns = Route53("Wildcard DNS\n*.bioc.cancerdatasci.org")
lb = LoadBalancing("Load Balancer")
pg = Postgresql("AWS Aurora PostgreSQL")
users = Users("Workshop\nParticipants")
web = Client("Standard Web Browser")
with Cluster("Kubernetes Cluster"):
app = Python("Orchestra")
master = Master("Kubernetes Master\nRunning on GKE")
k8api = API("Kubernetes Control API")
s = []
w = []
ing = Traefik("Router & Proxy")
ing >> app
app >> pg
app >> k8api
k8api >> master
pg >> app
prom = Prometheus("Prometheus\nMonitoring")
graf = Grafana("Grafana\nVisualization")
ing >> graf
graf >> prom
for i in range(0, 5):
from diagrams.onprem.iac import Terraform, Ansible
graph_attr = {"bgcolor": "transparent", "dpi": "300"}
with Diagram(
"Example Project",
show=True,
filename="clustered_web_services_example",
graph_attr=graph_attr,
):
with Cluster("Multi-Region"):
lb = ELB("ELB")
with Cluster("Services"):
autoscaling = AutoScaling("autoscaling-group")
svc_group = [EC2("web-01"), EC2("web-02")]
lb >> autoscaling >> svc_group
with Cluster("AMI"):
docker = Docker("Docker")
with Cluster("Docker Image"):
app = Python("App")
svc_group - docker
docker - Edge(style="dotted") - app
with Cluster("StreamState cluster"):
# svc_group = [ECS("web1"), ECS("web2"), ECS("web3")]
with Cluster("Replay"):
kafka_storage = Storage("Kafka sink")
spark_reload = Spark("Replay")
with Cluster("Realtime"):
spark_persist = Spark("No transforms")
spark_state = Spark("Analytical Stream")
argo = Argocd("Gitops")
argo >> spark_state
argo >> spark_reload
with Cluster("Dev"):
code = Code("Dev App")
code >> argo
code >> argo
firestore = Firestore("Cache/upsert")
spark_persist >> kafka_storage
kafka_storage >> spark_reload
kafka_input >> spark_state
kafka_input >> spark_persist
spark_state >> firestore
spark_reload >> firestore
spark_state >> kafka_output
spark_reload >> kafka_output
firestore >> Python("python sdk")
"""This module lets you re-create the dataflow diagram from the README."""
from diagrams import Cluster, Diagram
from diagrams.aws.analytics import EMRCluster
from diagrams.aws.storage import SimpleStorageServiceS3Bucket
from diagrams.onprem.analytics import Spark
from diagrams.programming.flowchart import Document, MultipleDocuments
from diagrams.programming.language import Python
with Diagram("Dataflow NLP-Weather-dataset", show=True):
with Cluster('Local input'):
local_input = [
MultipleDocuments('Yelp files'),
MultipleDocuments('GHCN files')
]
with Cluster('AWS'):
s3_bucket_input = SimpleStorageServiceS3Bucket('<your_s3_bucket>')
s3_bucket_output = SimpleStorageServiceS3Bucket('<your_s3_bucket>')
emr = EMRCluster('EMR')
spark = Spark('spark_app')
with Cluster('Local output'):
local_output = Document('nlp_weather_dataset')
local_input >> Python('local_to_s3') >> s3_bucket_input
s3_bucket_input >> emr >> spark >> s3_bucket_output
s3_bucket_output >> Python('s3_to_local') >> local_output
class Vault(Custom):
def __init__(self, label):
super(Vault, self).__init__(label, './img/vault.png')
class ExternalSecrets(Custom):
def __init__(self, label):
super(ExternalSecrets, self).__init__(label,
'./img/external-secrets.png')
with Diagram("Secret Management Flow",
graph_attr=graph_attr,
outformat="jpg",
show=False):
init_script = Python("vault_init.py")
init_secrets = Secret('Kubernetes secrets')
with Cluster("External-secrets components") as external_secrets_cluster:
external_service_account = ServiceAccount('Service account')
external_secrets = ExternalSecrets("Secret workers")
external_secrets_manifests = Secret('External secrets')
external_secrets << external_secrets_manifests
external_secrets << external_service_account
with Cluster("Vault components") as vault_cluster:
vault = Vault('vault')
vault_init_secrets = Secret('vault-init-secrets')
from diagrams import Cluster, Diagram
from diagrams.programming.language import Python
from diagrams.aws.compute import EC2
from diagrams.custom import Custom
graph_attr = {"bgcolor": "transparent", "pad": "0.5"}
with Diagram(name="",
show=False,
graph_attr=graph_attr,
filename="./docs/architecture_image"):
with Cluster("", graph_attr={"bgcolor": "azure3"}):
(EC2("Your API") >> Python("SCTS") >>
Custom("Correios",
icon_path="./images/correios-logo.png") # << Python("SCTS")
)