def main():
with Diagram("Home Network",
show=False,
filename="output/home_network",
outformat="png"):
internet = Internet("internet")
with Cluster("192.168.0.1/24"):
router = Router("Synology RT2600ac") # SynologyRouter
switch = Switch("Netgear R7000")
raspberrypi = Server("Raspberry Pi 4 8GB") # raspberrypi
raspberrypi_docker = Docker("Docker")
devices = Client("Devices")
internet >> router
internet << router
router >> raspberrypi
router >> switch
router >> devices
switch >> devices
with Cluster("10.0.0.0/28"):
service_nginx_proxy = Nginx("nginx-proxy")
service_grafana = Grafana("Grafana")
service_pi_hole = DNS("Pi-hole")
service_portainer = LinuxGeneral("Portainer")
service_telegraf = LinuxGeneral("Telegraf")
service_prometheus = Prometheus("Prometheus")
service_loki = Loki("Loki")
service_promtail = Loki("Promtail")
raspberrypi >> raspberrypi_docker
raspberrypi_docker >> service_nginx_proxy
service_nginx_proxy >> service_grafana
service_nginx_proxy >> service_pi_hole
service_nginx_proxy >> service_portainer
service_nginx_proxy >> service_telegraf
service_nginx_proxy >> service_prometheus
service_nginx_proxy >> service_loki
service_nginx_proxy >> service_promtail
service_prometheus >> Edge(label="collect metrics",
color="firebrick",
style="dashed") >> service_telegraf
service_promtail >> Edge(label="push logs",
color="firebrick",
style="dashed") >> service_loki
service_grafana >> Edge(label="query",
color="firebrick",
style="dashed") >> service_prometheus
service_grafana >> Edge(label="query",
color="firebrick",
style="dashed") >> service_loki
def generate_overview_diagram():
graph_attr = {
"bgcolor": "transparent"
}
with Diagram("Overview", show=False, filename="bin/overview", graph_attr=graph_attr):
with Cluster("Client"):
webapp = Angular("webapp")
with Cluster("API Services"):
status_service = Python("status_service")
task_service = Python("task_service")
worker_service = Python("worker_service")
metrics_service = Python("metrics_service")
with Cluster("Intermediate Services"):
with Cluster("Clearly Client Cluster"):
clearly_client = Python("clearly_client")
with Cluster("Backend Services"):
with Cluster("Clearly Server Cluster"):
clearly_server = Python("clearly_server")
with Cluster("External Connections"):
with Cluster("Message Broker"):
redis = Redis("redis")
with Cluster("Monitoring"):
grafana = Grafana("grafana")
prometheus = Prometheus("prometheus")
webapp >> status_service << clearly_client
webapp >> task_service << clearly_client
webapp >> worker_service << clearly_client
clearly_client >> clearly_server >> redis
metrics_service << prometheus
metrics_service >> clearly_server
with Cluster("Egress Router Namespace A"):
haproxy_egressA = Ingress("haproxy")
#with Cluster("Egress Router Namespace B"):
# haproxy_egressB= Ingress("haproxy")
with Cluster("Namespace A"):
#POD1 = Pod("frontend")
PODF = [Pod("frontend"), Pod("frontend"), Pod("frontend")]
PODB = [Pod("backend"), Pod("backend"), Pod("backend")]
haproxy_ingressA >> Edge(color="brown") >> Service(
"frontend") >> PODF >> Service("backend") >> PODB >> SQL(
"Replica1") >> haproxy_egressA >> serverExtA
with Cluster("Namespace Metrics"):
metrics = Prometheus("metric")
PODF >> metrics << Edge(color="firebrick",
style="dashed") << Grafana("monitoring")
with Cluster("Namespace Logging"):
PODF >> Fluentd("forwarder") >> Splunk("CentralLogging")
#with Cluster("Namespace B"):
# masterB = Pod("POD 3")
# masterB - Pod("POD 4") >> haproxy_egressB >> SQL("Replica1") >> serverExtB
# haproxy_ingressB >> Edge(color="black") >> masterB
#aggregator = Fluentd("logging")
#aggregator >> Edge(label="parse") >> Kafka("stream") >> Edge(color="black", style="bold") >> Spark("analytics")
# haproxy_ingress >> Edge(color="darkorange")
from diagrams.onprem.logging import Fluentd
from diagrams.onprem.monitoring import Grafana, Prometheus
from diagrams.onprem.network import Nginx
from diagrams.onprem.queue import Kafka
if len(sys.argv) > 1:
file = str(sys.argv[1])
else:
file = "diagram"
with Diagram("Advanced Web Service with On-Premise", filename=file,
show=False):
ingress = Nginx("ingress")
metrics = Prometheus("metric")
metrics << Grafana("monitoring")
with Cluster("Service Cluster"):
grpcsvc = [Server("grpc1"), Server("grpc2"), Server("grpc3")]
with Cluster("Sessions HA"):
master = Redis("session")
master - Redis("replica") << metrics
grpcsvc >> master
with Cluster("Database HA"):
master = PostgreSQL("users")
master - PostgreSQL("slave") << metrics
grpcsvc >> master
aggregator = Fluentd("logging")
with Diagram(show=False, outformat="png",
filename="diagrams/tbd",
direction="LR",
node_attr ={"fontsize": "14"},
edge_attr ={"labelfontsize": "14"}):
with Cluster("Google Cloud") as cloud:
storage = GCS("Google Cloud Storage")
with Cluster("Kubernetes cluster"):
with Cluster("Monitoring") as monitoring:
metrics = Prometheus("Prometheus")
metrics_push = Prometheus("Prometheus Gateway")
metrics_push << Edge(label="pull metrics") << metrics
service_monitor = CRD("ServiceMonitor")
service_monitor << Edge(label="watch") << metrics
metrics << Edge(label="query metrics") << Grafana("Grafana")
with Cluster("Apache Spark application") as spark:
Spark()
driver = Pod("driver")
driver << service_monitor
exec_1 = Pod("executor 1")
exec_2 = Pod("executor 2")
exec_3 = Pod("executor 3")
driver << exec_1
driver << exec_2
driver << exec_3
driver >> Edge(label="push metrics") >> metrics_push
driver << Edge(label="pull metrics") << metrics
exec_1 >> Edge(label="read/write") >> storage
exec_2 >> Edge(label="read/write") >> storage
exec_3 >> Edge(label="read/write") >> storage
users = Users("Workshop\nParticipants")
web = Client("Standard Web Browser")
with Cluster("Kubernetes Cluster"):
app = Python("Orchestra")
master = Master("Kubernetes Master\nRunning on GKE")
k8api = API("Kubernetes Control API")
s = []
w = []
ing = Traefik("Router & Proxy")
ing >> app
app >> pg
app >> k8api
k8api >> master
pg >> app
prom = Prometheus("Prometheus\nMonitoring")
graf = Grafana("Grafana\nVisualization")
ing >> graf
graf >> prom
for i in range(0, 5):
with Cluster(f"Workshop {i+1}"):
s.append(Service(f"Service"))
w.append(Pod(f"Rstudio"))
s[i] >> w[i]
ing >> s[i]
master - Edge(color="firebrick", style="dashed") - s[i]
master - Edge(color="firebrick", style="dashed") - w[i]
prom >> w[i]
lb >> ing
master - Edge(color="firebrick", style="dashed") - ing
users >> web
from diagrams import Cluster, Diagram, Edge
from diagrams.onprem.iac import Ansible
from diagrams.onprem.monitoring import Grafana, Prometheus
from diagrams.onprem.queue import RabbitMQ
from diagrams.programming.framework import Django
from diagrams.programming.language import Python
with Diagram("MQ Cluster", show=False, direction="TB"):
provisioner = Ansible("mgmt-core")
metrics = Grafana("monitoring") >> Edge(color="firebrick", style="dashed") >> Prometheus("rabbitmq_exporter")
producer = Django("Django App")
consumer = Python("Celery worker")
with Cluster("RabbitMQ 3-Node Cluster"):
mq_cluster = [RabbitMQ("leader"),
RabbitMQ("follower_1"),
RabbitMQ("follower_2")]
provisioner >> mq_cluster
producer >> Edge(color="brown") >> mq_cluster << Edge(color="brown") << consumer
mq_cluster << metrics
with Diagram("MQ Single", show=False, direction="TB"):
provisioner = Ansible("mgmt-core")
metrics = Grafana("monitoring") >> Edge(color="firebrick", style="dashed") >> Prometheus("rabbitmq_exporter")
producer = Django("Django App")
filename = os.path.splitext(os.path.basename(__file__))[0]
title = "Architectural nodes design"
direction = 'LR'
graph_attr = {"pad": "0"}
with Diagram(title,
filename=filename,
graph_attr=graph_attr,
outformat='png',
direction=direction,
show=False):
ingress = Nginx("ingress")
metrics = Prometheus("metric")
metrics << Edge(color="firebrick", style="dashed") << Grafana("monitoring")
with Cluster("Service Cluster"):
grpcsvc = [Server("grpc1"), Server("grpc2"), Server("grpc3")]
with Cluster("Sessions HA"):
primary = Redis("session")
primary - Edge(color="brown",
style="dashed") - Redis("replica") << Edge(
label="collect") << metrics
grpcsvc >> Edge(color="brown") >> primary
with Cluster("Database HA"):
primary = PostgreSQL("users")
primary - Edge(color="brown",
style="dotted") - PostgreSQL("replica") << Edge(
def main():
graph_attr = {
"fontsize": "45",
'overlap_scaling': '100',
'size': '24!',
'ratio': 'expand'
}
with Diagram(name='Automation Framework Compose',
direction='LR',
graph_attr=graph_attr):
with Cluster('Docker Cluster'):
docker = Docker('Docker')
with Cluster('container1'):
python_container = Python('APIs\nOther Microservices')
with Cluster('Docker Registry'):
docker_registry_container = Docker('Docker Registry\ntcp:5000')
with Cluster('Docker Registry Browser'):
docker_registry_browser_container = Python(
'Docker Registry Browser\ntcp:8088')
with Cluster('BatFish'):
batfish_container = Custom(
'BatFish\ntcp:8888\ntcp:9997\ntcp:9996',
'custom_icons/BatFish.png')
with Cluster('Kafka Cluster'):
with Cluster('Zookeeper'):
Zookeeper('Zookeeper\ntcp:2181')
with Cluster('REST Proxy'):
rest_proxy = Custom('REST Proxy\ntcp:8082',
'custom_icons/REST-API.png')
with Cluster('Control Center'):
control_center = Kafka('Control Center\ntcp:9021')
with Cluster('Schema Registry'):
schema_registry = Storage('Schema Registry\ntcp:8081')
with Cluster('Brokers'):
broker_1 = Kafka('Broker 1\ntcp:9092')
kafka_brokers = [
broker_1,
Kafka('Broker 2\ntcp:9093'),
Kafka('Broker 3\ntcp:9094')
]
with Cluster('Secrets Managers'):
vault = Vault('HashiCorp Vault\ntcp:8200')
secrets_managers = [
vault,
]
with Cluster('Logging and Search'):
with Cluster('ELK Stack'):
elastic_search = Elasticsearch('Elastic Search\ntcp:9200')
kibana = Kibana('Kibana\ntcp:5601')
logstash = Logstash('Logstash\ntcp:5044')
search_log = [elastic_search, kibana, logstash]
with Cluster('Influxdb'):
infulxdb = Influxdb('Influxdb\ntcp:8086')
with Cluster('Grafana'):
grafana = Grafana('Grafana\ntcp:3000')
with Cluster('Inventory and Connectivity'):
with Cluster('Inventory'):
nautobot = Custom('Nautobot\ntcp:8000',
'custom_icons/Nautobot.jpeg')
with Cluster('Database'):
with Cluster('Mongo dB'):
mongodb = Mongodb('MongoDb\ntcp:27017')
mongodb_express = Mongodb('MongoDb Express\ntcp:8181')
mongo_group = [mongodb, mongodb_express]
with Cluster('CI/CD'):
team_city = TC('TeamCity')
kafka_brokers - python_container
python_container - vault
python_container - nautobot
nautobot - logstash
python_container - logstash
nautobot - infulxdb
python_container - infulxdb
python_container - mongodb
from diagrams import Diagram
from diagrams import Cluster
from diagrams.gcp.network import LoadBalancing, DNS
from diagrams.gcp.database import BigTable, Memorystore
from diagrams.gcp.storage import Storage
from diagrams.gcp.compute import ComputeEngine
from diagrams.onprem.monitoring import Prometheus, Grafana
with Diagram("Pastebin architecture", show=False):
dns = DNS("dns")
elb = LoadBalancing("loadbalancer")
with Cluster("REST"):
gces = [ComputeEngine("gce") for _ in range(3)]
cache_m = Memorystore("cache")
with Cluster("NoSQL"):
db = BigTable("bigtable1")
store = Storage("filestore")
with Cluster("monitoring"):
prom = Prometheus("prometheus")
graf = Grafana("grafana")
dns >> elb
elb >> gces
elb >> graf
gces >> cache_m >> db
gces >> store >> gces
gces - prom
graf >> prom
thanos = Thanos("\nThanos")
prometheus - thanos
with Cluster("Storage", graph_attr=graph_attr):
with Cluster("Logs", graph_attr=graph_attr):
elasticsearch = Elasticsearch("\nElasticsearch")
solr = Solr("\nSolr")
mongodb = Mongodb("\nMongoDB")
elasticsearch - solr - mongodb
with Cluster("Metrics", graph_attr=graph_attr):
influx = Influxdb("\nInfluxDB")
prometheus2 = Prometheus("\nPrometheus")
prometheus2 - influx
loki >> elasticsearch
thanos >> prometheus2
with Cluster("Visualization", graph_attr=graph_attr):
kibana = Kibana("\nKibana")
grafana = Grafana("\nGrafana")
influx >> kibana
mongodb >> grafana
with Cluster("Cloud", graph_attr=graph_attr):
with Cluster("Azure", graph_attr=graph_attr):
azlog = LogAnalyticsWorkspaces("\nLog Analytics")
with Cluster("AWS", graph_attr=graph_attr):
awslog = Cloudwatch("\nCloudwatch")
grafana >> Edge(color="white") >> azlog
from diagrams import Diagram, Cluster
from diagrams.k8s.group import Namespace
from diagrams.k8s.compute import Deployment, Pod
from diagrams.k8s.network import Service
from diagrams.onprem.monitoring import Prometheus, Grafana
from diagrams.onprem.network import Istio
with Diagram('Observability') as result:
with Cluster('Minikube') as minikube:
with Cluster('default') as app_namespace:
Service('simpleapp-sevice') << Deployment('simpleapp') << Pod('simpleapp') << Istio('proxy')
with Cluster('kube-system') as kube_system:
istio = Istio('config')
istio >> Pod('Kiali')
prom = Prometheus()
prom << Grafana()
prom >> Pod('Jaegar')
if __name__ == "__main__":
result
from diagrams import Diagram, Cluster
from diagrams.aws.compute import EC2
from diagrams.onprem.monitoring import Grafana, Prometheus
from diagrams.onprem.network import Apache, Nginx
from diagrams.onprem.iac import Ansible
with Diagram("Project DevOps#1", show=True):
with Cluster(
'Project DevOps#1 Infrastructure in vagrant - VirtualBox VMs'):
with Cluster('web1'):
nginx = Nginx('Nginx')
with Cluster('Web2'):
apache = Apache('Apache2')
with Cluster('Monitoring'):
with Cluster('prom1'):
prom = Prometheus('Prometheus')
ans = Ansible('Ansible control node')
with Cluster('graf1'):
graf = Grafana('Grafana')
prom - graf
prom >> nginx
prom >> apache
(
ingress
>> Edge(color="green")
>> [
Server("adguard"),
Server("espial"),
Server("miniflux"),
PostgreSQL("miniflux_db"),
Server("firefly"),
PostgreSQL("firefly_db"),
Server("healthchecks"),
Server("n8n"),
PostgreSQL("n8n_db"),
Prometheus("prometheus"),
Grafana("grafana"),
Firewall("oauth"),
]
)
with Cluster("Media Node"):
media_services = [
Server("plex"),
Server("ombi"),
Server("calibre"),
Server("calibre-web"),
Server("nzbget"),
Server("radarr"),
Server("sonarr"),
Server("lazylibrarian"),
Server("tautulli"),
show=False,
direction="TB",
outformat="png",
graph_attr=graph_attr,
node_attr=node_attr):
with Cluster("Circle VPC"):
with Cluster("AWS services"):
aws_group = [
EC2("service01"),
EC2("service02"),
]
with Cluster("metrics host"):
metrics = Prometheus("metrics")
alertmanager = Prometheus("alertmanager")
dashboard = Grafana("monitoring")
metrics << dashboard
metrics >> alertmanager
Ansible("ansible") >> metrics
metrics >> Edge(style="dashed",
label="ec2 read permissions") >> General("AWS API")
alertmanager >> Edge(style="dashed",
label="non-critical") >> Slack("circle-alerts")
alertmanager >> Edge(style="dashed", label="critical") >> Pushover("tbd")
ELB("metrics.circlelinux.org") >> Edge(label="TCP3000") >> dashboard
with Cluster("Cloudvider"):
cloudvider_group = [
Server("server01"),
Server("server02"),
with Cluster("Message Search"):
es = Elasticsearch("Elasticsearch")
kfk >> Edge(color="blue", label="Elasticsearch Sink Connector") >> es
es << Edge(color="blue", label="Search") >> web_clients
with Cluster("Caching"):
with Cluster("Aiven"):
rds = Redis("Redis")
web_clients << Edge(color="yellow", label="Response") << rds << Edge(
color="yellow", label="Response") << hasura
# Step 5
with Diagram(show=True, filename=files[4]):
with Cluster("Metrics"):
metrics = InfluxDB("InfluxDB / M3")
graf = Grafana("Dashboards")
with Cluster("Web"):
web_clients = [React("Client 1"), React("Client 2")]
with Cluster("API and Database"):
with Cluster("Heroku"):
hasura = Server("GraphQL")
with Cluster("Aiven"):
with Cluster("Database HA"):
pg = PostgreSQL("DB")
pg - Edge(color="brown", style="dotted") - PostgreSQL(
"secondary") << Edge(label="collect") << metrics
web_clients << Edge(color="green") << hasura
hasura >> Edge(color="green") << pg
web_clients << Edge(color="green") >> hasura >> Edge(
color="green") << pg
with Diagram("OpenFlow with Monitoring"):
with Cluster("Backend"):
b = [Mongodb("MongoDB"), Rabbitmq("RabbitMQ")]
with Cluster("Remote Clients"):
rc = [Custom("OpenRPA", "./my_resources/open_rpa128.png"), Custom("PowerShell",
"./my_resources/PowerShell_5.0_icon.png"), Custom("NodeRED", "./my_resources/node-red-icon.png")]
with Cluster("Frontend + API"):
api = EC2("WEB-API")
cn = Custom("NodeRED", "./my_resources/node-red-icon.png")
with Cluster("Monitoring"):
g = Grafana("Grafana")
p = Prometheus("Prometheus")
otel = EC2("Open Telemetry")
j = Jaeger("Jaeger")
c = Cassandra("Cassandra")
t = Traefik("Traefik")
b << api
cn << t
api << t
t << rc
otel << b
otel << rc
otel << api
cloudflared = Custom("cloudflared", icons["cloudflareCloud"]["icon"])
traefik = Traefik("traefik")
with Cluster("Internal"):
visualizer = Custom("visualizer", icons["visualizer"]["icon"])
gitea = Gitea("Gitea")
allaboutsecurity = Custom("allaboutsecurity.xyz", icons["ghost"]["icon"])
ghost = Custom("Ghost", icons["ghost"]["icon"])
vscode = Custom("VSCode Server", icons["vscode"]["icon"])
firefly = Custom("Firefly III", icons["firefly"]["icon"])
fireflycsv = Custom("Firefly CSV", icons["firefly"]["icon"])
gaps = Custom("Gaps", icons["gaps"]["icon"])
grafana = Grafana("Grafana")
homeassistant = Custom("Home Assistant", icons["homeassistant"]["icon"])
netbox = Custom("NetBox", icons["netbox"]["icon"])
nodered = Custom("Node-Red", icons["nodered"]["icon"])
pihole = Custom("Pi-Hole", icons["pihole"]["icon"])
prometheus = Prometheus("Prometheus")
portainer = Custom("Portainer", icons["portainer"]["icon"])
speedtest = Custom("Speedtest", icons["speedtest"]["icon"])
spotifytrends = Custom("Spotify Trends", icons["spotify"]["icon"])
ssh = Custom("SSH", icons["ssh"]["icon"])
taskcafe = Custom("TaskCafe", icons["taskcafe"]["icon"])
youtubedl = Custom("YouTube-DL", icons["youtube"]["icon"])
with Cluster("VM#2"):
s3 = [Server("S3 - A...Z")]
# Volumes
with Cluster("Files"):
volumeA = FileStorage("/files")
volumeDB = Database("/database")
# Logging
with Cluster("Logging"):
logs = FluentBit("logging")
s2 >> Edge(label="produces") >> logs
# Monitoring
with Cluster("Monitoring"):
s2 << Edge(label="collect") << Prometheus("metric") << Grafana(
"monitoring")
httpsProxy = Nginx("HTTPS_PROXY")
with Cluster("Cloud APIs"):
cAPI_ELB = ELB("Cloud API - ELB")
cAPI_Targets = [ECS("Cloud API - Organization(s) A...Z")]
cAPI_ELB >> cAPI_Targets
s1 >> Edge(label="produces") >> volumeA
s2 >> Edge(label="consumes") >> volumeA
s2 >> Edge(label="R/W") >> volumeDB
# Configuration
s2 >> Edge(label="Read/Write") >> configServerP
return svc
with Diagram("Lorem Ipsum Cloud Native", show=True, filename='lorem_ipsum_cloud_native'):
ui = React("UI")
with Cluster("Kube"):
net = Istio("loremipsum.com")
with Cluster('Logging'):
logging = Fluentd('logging')
es = ElasticSearch('index')
kibana = Kibana('Log View')
logging >> es >> kibana
with Cluster('Monitoring'):
metrics = Prometheus('metrics')
monitoring = Grafana('monitoring')
tracing = Jaeger('tracing')
with Cluster("DB") as cluster:
db_books = PostgreSQL("books")
db_auth = PostgreSQL("auth")
# pvc = PVC("pvc")
# pvc << [db_books, db_auth]
with Cluster('Lorem Ipsum Apps'):
books_svc = Service('books')
apps = []
for _ in range(2):
pod = Pod("pod")
apps.append(books_svc >> pod >> [db_books, metrics, logging])
auth_svc = Service('auth')
apps = []
with Cluster("Edge Stack", direction="LR"):
edge_stack = Ambassador("Ambassador Ingress Gateway")
edge_stack >> [Kong("KONG API Gateway")]
devops_team >> edge_stack
with Cluster("Service Discovery and Mesh"):
service_discovery_mesh = Consul("Consul")
service_discovery_mesh << [Envoy("Consul Connect+Envoy")]
devops_team >> service_discovery_mesh
with Cluster("Monitoring and Alerting"):
monitoring = Prometheus("Prometheus Metric and Alert Manager")
monitoring >> [Grafana("Dashboard Monitoring")] << openldap
devops_team >> monitoring
with Cluster("Scheduler and Operational Task"):
rundeck = Custom("Rundeck", "./resources/rundeck.png")
rundeck << [openldap]
devops_team >> rundeck
with Cluster("VPN"):
vpn = Custom("Pritunl", "./resources/pritunl.png")
vpn << [openldap]
devops_team >> vpn