def get_common_architecture(cloud_provider, filename, container_registry,
k8s_cluster_name, k8s_engine_name):
with Diagram(
f"QHub Architecture: {cloud_provider}",
filename=f"{filename}",
show=False,
direction="TB",
):
with Cluster(cloud_provider):
gcr = ContainerRegistry(container_registry)
with Cluster(k8s_cluster_name):
kube_engine = KubernetesEngine(k8s_engine_name)
conda_pvc = PersistentVolume("Conda")
nfs_pvc = PersistentVolume("NFS")
elb = LoadBalancing("Ingress")
with Cluster("Master"):
general = Node("general")
Node("user")
Node("worker")
general << kube_engine
with Cluster("Pods"):
dask_gateway = Pod("Dask Gateway")
jupyterhub = Pod("JupyterHub")
dask_scheduler = Pod("Dask Scheduler")
nfs_server = Pod("NFS Server")
conda_store = Pod("Conda Store")
nginx = Pod("Nginx")
Pod("Cert Manager")
image_puller = Pod("Image Puller")
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
image_puller >> gcr
nfs_server >> nfs_pvc
conda_store >> conda_pvc
Helm("Helm") >> kube_engine
def get_common_architecture(cloud_provider, filename, container_registry,
k8s_cluster_name, k8s_engine_name):
with Diagram(
f"QHub Architecture: {cloud_provider}",
filename=f'{filename}',
show=False,
direction="TB",
):
with Cluster(cloud_provider):
gcr = ContainerRegistry(container_registry)
with Cluster(k8s_cluster_name):
kube_engine = KubernetesEngine(k8s_engine_name)
conda_pvc = PersistentVolume('Conda')
nfs_pvc = PersistentVolume('NFS')
elb = LoadBalancing('Ingress')
with Cluster('Master'):
general = Node('general')
Node('user')
Node('worker')
general << kube_engine
with Cluster("Pods"):
dask_gateway = Pod('Dask Gateway')
jupyterhub = Pod('JupyterHub')
dask_scheduler = Pod('Dask Scheduler')
nfs_server = Pod('NFS Server')
conda_store = Pod('Conda Store')
nginx = Pod('Nginx')
Pod('Cert Manager')
image_puller = Pod('Image Puller')
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
image_puller >> gcr
nfs_server >> nfs_pvc
conda_store >> conda_pvc
Helm('Helm') >> kube_engine
from diagrams.gcp.database import Datastore, Memorystore, SQL
from diagrams.gcp.network import LoadBalancing
from diagrams.onprem.client import User
os.chdir(os.path.dirname(__file__))
with Diagram(
"Notebook to TAP",
show=False,
filename="notebook-tap",
outformat="png",
):
user = User("End User")
with Cluster("Kubernetes"):
ingress = LoadBalancing("NGINX Ingress")
with Cluster("Nublado"):
hub = KubernetesEngine("Hub")
proxy = KubernetesEngine("Proxy")
notebook = KubernetesEngine("Notebook")
proxy >> hub
hub >> notebook
proxy >> notebook
with Cluster("TAP"):
tap = KubernetesEngine("TAP")
schema = SQL("MySQL Schema DB")
uws = SQL("PostgreSQL UWS")
with Cluster("Gafaelfawr"):
def draw():
with Diagram("Example Microservice for EBA", show=False, graph_attr=graph_attr, direction="LR"):
staff = Client('EBA Staff')
organinzer = User("EBA Organinzer")
users = Users("EBA Users")
mobileApp = Flutter("EBA Mobile")
dns = DNS('Cloud DNS')
loadbalance = LoadBalancing('Cloud Loadbalancing')
auth = Authentication("Firebase Auth")
users >> Edge(color="darkgreen") >> mobileApp << Edge(
color="darkgreen") >> auth
[organinzer, staff] << Edge(color="darkgreen") >> dns
dns << Edge(color="darkgreen") >> loadbalance
with Cluster("Google Cloud Platform", graph_attr=graph_attr2):
with Cluster("WEB Frontend", graph_attr=graph_attr2):
with Cluster("Web Container1", graph_attr=graph_attr2):
web1 = Hosting("Firebase Hosting")
react1 = React("Frontend")
web1 - react1
with Cluster("Web Container2", graph_attr=graph_attr2):
web2 = Hosting("Firebase Hosting")
react2 = React("Frontend")
web2 - react2
with Cluster("WEB Backend", graph_attr=graph_attr2):
with Cluster("Web Backend Container"):
web3 = Hosting("Firebase Hosting")
react3 = React("Backend")
web3 - react3
loadbalance << Edge(color="darkgreen") >> [web1, web2, web3]
with Cluster("APIs", graph_attr=graph_attr2):
func = Functions("Cloud Functions")
remoteCnf = RemoteConfig("RemoteConfig")
[react1, react2, react3] << Edge(
color="darkgreen") >> remoteCnf << Edge(
color="darkgreen") >> func
with Cluster("Feeds", graph_attr=graph_attr2):
apiFeeds = NodeJS("Express")
with Cluster("Tournaments", graph_attr=graph_attr2):
apiTournament = NodeJS("Express")
with Cluster("Badges", graph_attr=graph_attr2):
apiBadges = NodeJS("Express")
with Cluster("User", graph_attr=graph_attr2):
apiUser = NodeJS("Express")
with Cluster("Games", graph_attr=graph_attr2):
apiGames = NodeJS("Express")
with Cluster("Notifications", graph_attr=graph_attr2):
apiNotifications = NodeJS("Express")
auth << Edge(color="darkgreen") >> func >> Edge(color="darkorange") >> [apiBadges, apiUser, apiFeeds,
apiGames, apiNotifications, apiTournament]
with Cluster("Database", graph_attr=graph_attr2):
badgesSQL = SQL("Badges")
userSQL = SQL("User")
notificationSQL = SQL("Notifications")
tournamentsSQL = SQL("Tournaments")
gamesSQL = SQL("Games")
feedsSQL = SQL("Feeds")
apiBadges - Edge(color="darkorange") - badgesSQL
apiUser - Edge(color="darkorange") - userSQL
apiFeeds - Edge(color="darkorange") - feedsSQL
apiGames - Edge(color="darkorange") - gamesSQL
apiNotifications - Edge(color="darkorange") - notificationSQL
apiTournament - Edge(color="darkorange") - tournamentsSQL
firestore = Firestore("Firestore")
with Cluster("Message Queue", graph_attr=graph_attr2):
messageQueue = Pubsub("Pubsub")
[badgesSQL, userSQL, notificationSQL, tournamentsSQL,
gamesSQL, feedsSQL] >> Edge(color="darkorange") >> messageQueue
messageQueue >> Edge(color="darkred") >> firestore >> Edge(color="darkred", style="dotted") >> [apiBadges, apiUser, apiFeeds,
apiGames, apiNotifications, apiTournament]
with Cluster("Storage", graph_attr=graph_attr2):
badgesStorage = Storage("Badges")
userStorage = Storage("User")
gameStorage = Storage("Games")
tournamentsStorage = Storage("Tournaments")
apiBadges - Edge(color="blue") - badgesStorage
apiUser - Edge(color="blue") - userStorage
apiGames - Edge(color="blue") - gameStorage
apiTournament - Edge(color="blue") - tournamentsStorage
fcm = Messaging("Cloud Messaging")
apiNotifications >> \
Edge(color="darkorange") >> \
fcm >> Edge(color="darkgreen") >> auth
from diagrams import Cluster, Diagram
from diagrams.gcp.network import Router, LoadBalancing, DNS
from diagrams.k8s.compute import Pod
from diagrams.k8s.network import Ingress
from diagrams.generic.blank import Blank
graph_attr = {
# "splines": "curved"
}
with Diagram("OCP4 Default Ingress",
show=False,
direction="LR",
graph_attr=graph_attr):
dns = DNS("*.apps.dta.my.lab")
lbint = LoadBalancing("HAProxy")
with Cluster("OpenShift Cluster"):
routerext = Ingress("default")
with Cluster("Workers"):
with Cluster("Projects"):
with Cluster("proj-app-01"):
workersint = [Pod("myApp")]
with Cluster("proj-app-02"):
workersext = [Pod("myApp")]
dns >> lbint
lbint >> routerext
routerext >> workersint
def custom_icon(filename):
return os.path.join("architecture/icons", filename)
with Diagram(
"QHub Architecture: High Level",
filename="high_level_architecture",
show=False,
direction="TB",
):
with Cluster("Kubernetes Cluster"):
conda_pvc = PersistentVolume("Conda (Persistent Volume)")
nfs_pvc = PersistentVolume("NFS (Persistent Volume)")
elb = LoadBalancing("Ingress")
with Cluster("Pods"):
dask_gateway = Custom("Dask Gateway", custom_icon("dask.png"))
jupyterhub = Custom("JupyterHub", custom_icon("jupyter_hub.png"))
dask_scheduler = Custom("Dask Scheduler", custom_icon("dask.png"))
nfs_server = Pod("NFS Server")
conda_store = Custom("Conda Store", custom_icon("conda.png"))
nginx = Custom("Nginx", custom_icon("nginx.png"))
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
nfs_server >> nfs_pvc
conda_store >> conda_pvc
from diagrams import Diagram, Cluster
from diagrams.gcp.database import Firestore
from diagrams.gcp.network import CDN, DNS, LoadBalancing
from diagrams.gcp.storage import GCS
from diagrams.gcp.compute import GKE
from diagrams.onprem.compute import Server
from diagrams.oci.network import Internetgateway
with Diagram("Gran - GCP Design", show=False):
server = Server("Client (Nuxt.js)")
with Cluster("Google Cloud Platform"):
firestore = Firestore("Firebase Firestore")
cdn = CDN("Cloud CDN")
dns = DNS("Cloud DNS")
lb = LoadBalancing("Cloud Load Balancing")
gcs = GCS("Cloud Storage")
gke = GKE("Kubernetes Engine")
net = Internetgateway("Gateway")
dns >> cdn >> gcs
dns >> lb >> gke >> firestore
server >> net
net >> dns
from diagrams import Cluster, Diagram
from diagrams.gcp.compute import GKE, GCF
from diagrams.gcp.network import LoadBalancing
from diagrams.gcp.database import SQL
from diagrams.gcp.analytics import PubSub
with Diagram("GKE Cluster", show=True):
lb = LoadBalancing("External Load Balancer")
with Cluster("Default"):
pubsub = PubSub("PubSub")
with Cluster("Internal"):
gke = GKE("GKE")
lb >> gke >> pubsub
with Cluster("Backend"):
mysql = SQL("MYSQL")
gcf = GCF("Functions")
pubsub >> gcf >> mysql
instance_3 = ComputeEngine('Server 3')
disk_3 = PersistentDisk('OS Disk 3')
pv_disk = PersistentDisk('SSD Disk 1')
node_1 - instance_1
node_2 - instance_2
node_3 - instance_3
instance_1 - disk_1
instance_2 - disk_2
instance_3 - disk_3
instance_3 - pv_disk
pv_workspace - pv_disk
core - SQL('PostgreSQL')
core - GCS('Object Storage')
lb = LoadBalancing('Load Balancer')
dns = DNS('DNS')
lb >> ing
dns >> lb
[Client('SDKs'), Users('Users')] >> dns
with Diagram('Amazon Web Services resources',
show=False,
filename='aws',
outformat='png'):
with Cluster('Amazon Web Services'):
with Cluster('Virtual Private Cloud'):
kube = EKS('Elastic Kubernetes\nService')
instance_1 = EC2('EC2 Instance 1\n(m5.xlarge)')
disk_1 = EBS('EBS Disk 1\n(gp2)')
from diagrams import Diagram
from diagrams import Cluster
from diagrams.gcp.network import LoadBalancing, DNS
from diagrams.gcp.database import BigTable, Memorystore
from diagrams.gcp.storage import Storage
from diagrams.gcp.compute import ComputeEngine
from diagrams.onprem.monitoring import Prometheus, Grafana
with Diagram("Pastebin architecture", show=False):
dns = DNS("dns")
elb = LoadBalancing("loadbalancer")
with Cluster("REST"):
gces = [ComputeEngine("gce") for _ in range(3)]
cache_m = Memorystore("cache")
with Cluster("NoSQL"):
db = BigTable("bigtable1")
store = Storage("filestore")
with Cluster("monitoring"):
prom = Prometheus("prometheus")
graf = Grafana("grafana")
dns >> elb
elb >> gces
elb >> graf
gces >> cache_m >> db
gces >> store >> gces
gces - prom
graf >> prom
def main():
graph_attr = {"fontsize": "45", "bgcolor": "white"}
with Diagram("Deployment-0",
graph_attr=graph_attr,
show=False,
filename="img/deployment-0",
outformat="png",
direction="LR"):
with Cluster("Backup"):
velero = Backup("Velero")
with Cluster("Storage"):
with Cluster("OpeneEBS"):
with Cluster("node from pod"):
node = Server("node n")
with Cluster("Pod + Volume"):
hostpath = Pod("openebs-hostpath")
hostpath_pv = PV("LocalPV")
hostpath >> PVC("claim") >> hostpath_pv >> Edge(
color="darkred") >> node
hostpath >> node
velero >> Edge(color="darkred", style="dashed") << hostpath_pv
with Cluster("Longhorn"):
with Cluster("Storage nodes"):
nodes = [
Server("node-3"),
Server("node-2"),
Server("node-1")
]
with Cluster("Pod + Volume"):
longhorn = Pod("longhorn")
longhorn_pv = PV("Replicated")
longhorn >> PVC("claim") >> longhorn_pv >> Edge(
color="blue") >> nodes
velero >> Edge(color="blue", style="dashed") << longhorn_pv
with Cluster("Internet"):
internet = Internet("HTTP clients")
certprovider = Internet("Cert provider")
lan = Server("LAN")
s3 = S3("S3")
s3 >> Edge(color="blue") << velero
with Cluster("Ingress"):
with Cluster("MetalLB"):
metallb1 = LoadBalancing("IP1")
metallb2 = LoadBalancing("IP2")
certprovider >> Edge(color="darkred") >> metallb1
with Cluster("NGINX"):
ingress_ext = Nginx("External")
ingress_int = Nginx("Internal")
with Cluster("Certificates"):
certmanager = CertificateManager("cert-manager")
certissuer = CertificateManager("cert-issuer")
certmanager >> certissuer
ingress_ext >> Edge(color="red", label="acme") >> certmanager
ingress_ext << Edge(color="red",
label="cert-secret") << certmanager
internet >> Edge(color="blue") >> metallb1 >> Edge(
color="darkgreen") >> ingress_ext
certprovider << Edge(color="red", style="dashed",
label="http-01") << certissuer
lan >> Edge(color="blue") >> metallb2 >> Edge(
color="darkgreen") >> ingress_int
"""Notional system architecture."""
from diagrams import Cluster, Diagram
from diagrams.gcp.network import FirewallRules, LoadBalancing
from diagrams.k8s.compute import Deployment, Pod, ReplicaSet, StatefulSet
from diagrams.k8s.podconfig import Secret
from diagrams.k8s.network import Ingress, Service
from diagrams.k8s.storage import PersistentVolume, PersistentVolumeClaim, StorageClass
from diagrams.onprem.client import Client
with Diagram("TRM Exercise Detail", show=False):
with Cluster("GCP"):
# Resources
app_lb_ext = LoadBalancing("App LB (HTTP/external)")
app_lb_int = LoadBalancing("App LB (TCP/internal)")
app_ingress_ext = Ingress("App Ingress (external)")
app_fw = FirewallRules("Firewall")
with Cluster("GKE"):
with Cluster("App Namespace"):
# Resources
app_dep = Deployment("App Deployment")
app_rs = ReplicaSet("App ReplicaSet")
app_replica_set_count = 3
app_pods = [
Pod("App Pod")
for pod in range(0, app_replica_set_count, 1)
]
app_secret = Secret(
"App Secret\n(contains DB credentials & host)")
app_svc_ext = Service("App Service (external)")
from diagrams import Diagram, Cluster, Edge
from diagrams.aws.network import Route53
from diagrams.onprem.database import Postgresql
from diagrams.gcp.network import LoadBalancing
from diagrams.onprem.monitoring import Prometheus, Grafana
from diagrams.onprem.client import Users, Client
from diagrams.k8s.compute import Pod
from diagrams.k8s.infra import Master
from diagrams.k8s.controlplane import API
from diagrams.k8s.network import Ingress, Service
from diagrams.onprem.network import Traefik
from diagrams.programming.language import Python
with Diagram("Simplified Orchestra Architecture Diagram", show=False):
dns = Route53("Wildcard DNS\n*.bioc.cancerdatasci.org")
lb = LoadBalancing("Load Balancer")
pg = Postgresql("AWS Aurora PostgreSQL")
users = Users("Workshop\nParticipants")
web = Client("Standard Web Browser")
with Cluster("Kubernetes Cluster"):
app = Python("Orchestra")
master = Master("Kubernetes Master\nRunning on GKE")
k8api = API("Kubernetes Control API")
s = []
w = []
ing = Traefik("Router & Proxy")
ing >> app
app >> pg
app >> k8api
k8api >> master
pg >> app
}
with Diagram(
"A&A",
show=False,
filename="architecture",
outformat="png",
graph_attr=graph_attr,
):
user = User("End User")
with Cluster("Science Platform"):
filestore = Filestore("POSIX Filesystem")
with Cluster("Authentication"):
auth_ingress = LoadBalancing("NGINX Ingress")
auth_server = KubernetesEngine("Server")
storage = SQL("Metadata Store")
redis = KubernetesEngine("Redis")
redis_storage = PersistentDisk("Redis Storage")
user >> auth_ingress >> auth_server >> redis >> redis_storage
auth_server >> storage
with Cluster("Notebook Aspect"):
notebook_ingress = LoadBalancing("NGINX Ingress")
hub = KubernetesEngine("Hub")
session_storage = SQL("Session Storage")
notebook = KubernetesEngine("Notebook")
auth_server << notebook_ingress
def custom_icon(filename):
return os.path.join('architecture/icons', filename)
with Diagram(
"QHub Architecture: High Level",
filename='high_level_architecture',
show=False,
direction="TB",
):
with Cluster('Kubernetes Cluster'):
conda_pvc = PersistentVolume('Conda (Persistent Volume)')
nfs_pvc = PersistentVolume('NFS (Persistent Volume)')
elb = LoadBalancing('Ingress')
with Cluster("Pods"):
dask_gateway = Custom('Dask Gateway', custom_icon('dask.png'))
jupyterhub = Custom('JupyterHub', custom_icon('jupyter_hub.png'))
dask_scheduler = Custom('Dask Scheduler', custom_icon('dask.png'))
nfs_server = Pod('NFS Server')
conda_store = Custom('Conda Store', custom_icon('conda.png'))
nginx = Custom('Nginx', custom_icon('nginx.png'))
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
nfs_server >> nfs_pvc
conda_store >> conda_pvc