def test_validate_direction(self):
# Normal directions.
for dir in ("TB", "BT", "LR", "RL"):
with Diagram(name=os.path.join(self.name, "validate_direction"),
show=False):
Cluster(direction=dir)
# Invalid directions.
for dir in ("BR", "TL", "Unknown"):
with self.assertRaises(ValueError):
with Diagram(name=os.path.join(self.name,
"validate_direction"),
show=False):
Cluster(direction=dir)
def main():
with Diagram('New SFTP Process') as diag:
user = Windows('User')
with Cluster('Repay-CDE-PROD-SFTP', direction='TB'):
sftp_service = TransferForSftp('SFTP')
s3_sftp_bucket = S3('channels_sftp_transfer')
sftp_service - s3_sftp_bucket
aws_sftp_group = [sftp_service, s3_sftp_bucket]
user >> Edge(label='Upload a BLF into SFTP') >> sftp_service
with Cluster('repay-cde-prod-channels', direction='TB'):
lambda_blf_copy = Lambda('s3_copy_lambda')
s3_blf_processor = S3('blf_processor bucket')
sns_blf_uploaded_to_s3 = SNS('SNS - blf_uploaded_to_s3')
redis = ElastiCache('(redis)')
elasticsearch = ES('1 index per BLF')
lambda_blf_copy >> s3_blf_processor
s3_blf_processor >> sns_blf_uploaded_to_s3
cde_group = [
lambda_blf_copy, s3_blf_processor, sns_blf_uploaded_to_s3,
redis, elasticsearch
]
with Cluster('repay-cde-prod-k8s', direction='TB'):
k8s_api_pod = Pod('Channels API\n/api/v1/blf_upload_sns')
k8s_blf_processor_job = Job('process-blf-k8s')
k8s_api_pod >> Edge(
label='Create job if no BLF lock exists for org/filename'
) >> k8s_blf_processor_job
k8s_group = [k8s_api_pod, k8s_blf_processor_job]
# TODO - MAKE SURE TO HIGHLIGHT THE USE OF INVISIBLE EDGES
s3_sftp_bucket >> Edge(style='invis') >> cde_group
redis >> Edge(style='invis') >> k8s_group
elasticsearch >> Edge(style='invis') >> k8s_group
k8s_blf_processor_job << Edge(
label='Download file from s3') << s3_blf_processor
s3_sftp_bucket >> Edge(
label='S3 notification Object Created') >> lambda_blf_copy
sns_blf_uploaded_to_s3 >> Edge(
label='HTTP request with BLF file name') >> k8s_api_pod
k8s_api_pod >> Edge(
label='Create BLF lock - 5 minute expiration') >> redis
k8s_blf_processor_job >> Edge(
label='Delete BLF lock when done') >> redis
k8s_blf_processor_job >> Edge(label='Create ES index') >> elasticsearch
def test_node_to_nodes(self):
with Diagram(name=os.path.join(self.name, 'node_to_nodes'),
show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(node1 - Edge(color='red') - nodes, nodes)
def test_with_global_context(self):
with Diagram(name=os.path.join(self.name, 'with_global_context'),
show=False):
self.assertIsNone(getcluster())
with Cluster():
self.assertIsNotNone(getcluster())
self.assertIsNone(getcluster())
def render_transitions(dialogs: Union[List[Dialog], DialogRegistry],
title: str = "Aiogram Dialog",
filename: str = "aiogram_dialog",
format: str = "png"):
if isinstance(dialogs, DialogRegistry):
dialogs = list(dialogs.dialogs.values())
with Diagram(title, filename=filename, outformat=format, show=False):
nodes = {}
for dialog in dialogs:
with Cluster(dialog.states_group_name()):
for window in dialog.windows.values():
nodes[window.get_state()] = Custom(icon_path=ICON_PATH, label=window.get_state().state)
starts = []
for dialog in dialogs:
for window in dialog.windows.values():
starts.extend(find_starts(window.get_state(), [window.keyboard]))
for dialog in dialogs:
for window in dialog.windows.values():
walk_keyboard(nodes, dialog, starts, window.get_state(), [window.keyboard])
preview_add_transitions = getattr(window, "preview_add_transitions", None)
if preview_add_transitions:
walk_keyboard(nodes, dialog, starts, window.get_state(), preview_add_transitions)
def linking_nodes_diagram():
from diagrams import Diagram, Cluster
from diagrams.aws.compute import EC2
from diagrams.aws.network import ELB
from diagrams.aws.network import Route53
from diagrams.onprem.database import PostgreSQL # Would typically use RDS from aws.database
from diagrams.onprem.inmemory import Redis # Would typically use ElastiCache from aws.database
with Diagram(
"Simple Website Diagram", direction='LR'
) as diag4: # It's LR by default, but you have a few options with the orientation
dns = Route53("DNS")
load_balancer = ELB("Load Balancer")
database = PostgreSQL("User Database")
cache = Redis("Cache")
with Cluster("Webserver Cluster"):
svc_group = [
EC2("Webserver 1"),
EC2("Webserver 2"),
EC2("Webserver 3")
]
dns >> load_balancer >> svc_group
svc_group >> cache
svc_group >> database
print(
diag4
) # This will illustrate the diagram if you are using a Google Colab or Jypiter notebook.
def test_with_global_context(self):
with Diagram(name=os.path.join(self.name, "with_global_context"),
show=False) as d1:
self.assertIsNone(getcluster())
with Cluster():
self.assertIsNotNone(getcluster())
self.assertEqual(d1, getcluster())
def test_nodes_to_node(self):
with Diagram(name=os.path.join(self.name, "nodes_to_node"),
show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(nodes - Edge(color="red") - node1, node1)
def design(self):
s = " with Cluster(\"" + self.__service_name + " service\"):" + '\n'
with Cluster(self.__service_name + " service"):
s = s + "\t\t" + self.__image_name + " = Server(\"" + self.__image_name + "\")\n"
img = Server(self.__image_name)
self.node = img
return s
def hltd_bu():
with Diagram("HLTD BU Machines",
show=False,
graph_attr=graph_attr,
node_attr=node_attr,
outformat="pdf"):
with Cluster("HLT daemon service ",
graph_attr={"bgcolor": "#EBF3E7"}):
demon = nSE("/usr/lib/systemd/system/hltd.service")
# '/fff/ramdisk'
with Cluster("/opt/hltd/scratch/python/hltd.py "):
hltd = nA(
"MountManager()\nMountManager.submount_size('/fff/ramdisk')\nSystemMonitor()\nResourceRanger()"
)
with Cluster("/opt/hltd/scratch/python/SystemMonitor.py"):
SystemMonitor = nA("")
with Cluster("/opt/hltd/python/ResourceRanger.py "):
ResourceRanger = nA(
"watch /fff/ramdisk/appliance/boxes/\n watch /etc/appliance/dqm_resources/ \n IN_CREATE call findRunAndNotify()->\n->Resource.NotifyNewRun()->\n->Notify FU to start a run, HTTPConnection"
)
with Cluster("/opt/hltd/python/RunRanger.py"):
RunRanger = nA(
"watch /fff/data\n for new run:\n cleanupBUCmd(...)\ncreate Run(...)\nRun.AcquireResources()\nRun.Start()"
)
with Cluster("/opt/hltd/python/Run.py "):
run = nA(
"Run.__init__(): create anelasticDQM() \n Run.Start() -> Run.StartOnResource() -> \n -> OnlineResource.StartNewProcess()"
)
with Cluster("/opt/hltd/python/inotifywrapper.py",
graph_attr={"bgcolor": "#ECE8F6"}):
inotify = nA("inotify wrapper")
with Cluster("/opt/hltd/python/anelasticDQM.py",
graph_attr={"bgcolor": "#FDF7E3"}):
anel = nA(
"create DQMFileWatcherBU(runNumber)\n watch /fff/output/DQMOutput \n watch /fff/ramdisk/appliance/boxes/\n if all heartbeat files older 1m -> \n -> run is over"
)
bu = nL("FU machine \n /fff/output/DQMOutput")
boxes = nL("/fff/BU0/ramdisk/appliance/boxes/")
demon >> hltd
hltd >> ResourceRanger
hltd >> RunRanger
bu >> anel
hltd >> SystemMonitor >> boxes >> anel
ResourceRanger - Edge(color="brown", style="dashed") - inotify
RunRanger - Edge(color="brown", style="dashed") - inotify
RunRanger >> run >> anel
anel - Edge(color="brown", style="dashed") - inotify
def generate(name, num_of_instances):
with Diagram(name, direction='TB', show=True):
with Cluster('Auto Scaling Group'):
app_group = [EC2Instance('App') for _ in range(num_of_instances)]
rds_master = RDS('DB (Master)')
rds_standby = RDS('DB (Standby)')
Users('Users') >> Route53('Route 53') >> ALB('ALB') >> app_group >> rds_master
rds_master - rds_standby
def general(c):
with Diagram('general', show=False):
_ = NS('astrid-kube')
_cb_pod = Pod('CB')
_cb_deploy = Deployment('CB')
with Cluster('Services'):
_srvs = [
Service('elasticsearch-service'),
Service('kafka-service'),
Service('cb-manager-service')
]
with Cluster('Storage'):
_strg = PVC('elasticsearch-pv-volume') >> PV('elasticsearch-pv')
_srvs >> _cb_pod << _cb_deploy << _strg
def test_node_to_node(self):
with Diagram(name=os.path.join(self.name, "node_to_node"), show=False):
with Cluster():
node1 = Node("node1")
node2 = Node("node2")
self.assertEqual(node1 - node2, node2)
self.assertEqual(node1 >> node2, node2)
self.assertEqual(node1 << node2, node2)
def test_nodes_to_node(self):
with Diagram(name=self.name, show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(nodes - node1, node1)
self.assertEqual(nodes >> node1, node1)
self.assertEqual(nodes << node1, node1)
def k8s(name):
with Cluster(name):
with Cluster('ns: onepanel'):
svc_core = Service('core')
pd_core = Pod('core-*')
dep_core = Deployment('core')
dep_core >> pd_core
svc_core >> pd_core
svc_core_ui = Service('core-ui')
pd_core_ui = Pod('core-ui-*')
dep_core_ui = Deployment('core-ui')
dep_core_ui >> pd_core_ui
svc_core_ui >> pd_core_ui
with Cluster('ns: istio-system'):
ing = Ingress('istio-ingressgateway')
# cert = Secret('TLS')
with Cluster('ns: my-project'):
with Cluster('Workspace'):
svc_workspace = Service('jupyterlab')
pd_workspace = Pod('jupyterlab-*')
pv_workspace = PV('jupyterlab-data-0')
sts_workspace = StatefulSet('jupyterlab')
sts_workspace >> pd_workspace
pd_workspace - pv_workspace
svc_workspace >> pd_workspace
# with Cluster('ns: cert-manager'):
# certmanager = Pod('cert-manager')
# certmanager >> cert
ing >> [svc_core, svc_core_ui, svc_workspace]
node_1 = Node('node-1')
node_2 = Node('node-2')
node_3 = Node('node-2')
pd_core - node_1
pd_core - node_2
pd_core_ui - node_1
pd_core_ui - node_2
pd_workspace - node_3
return node_1, node_2, node_3, ing, pd_core, pv_workspace
def test_nodes_to_node_with_attributes_bothdirectional(self):
with Diagram(name=os.path.join(
self.name, 'nodes_to_node_with_attributes_bothdirectional'),
show=False) as diagram:
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(
nodes << Edge(color='green', label='4') >> node1, node1)
def test_nodes_to_node_with_attributes_bidirectional(self):
with Diagram(name=os.path.join(
self.name, "nodes_to_node_with_attributes_bidirectional"),
show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(
nodes << Edge(color="blue", label="5") >> node1, node1)
def test_nodes_to_node_with_additional_attributes(self):
with Diagram(name=os.path.join(
self.name, "nodes_to_node_with_additional_attributes"),
show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(
nodes - Edge(color="red") - Edge(color="green") - node1,
node1)
def create_entities(entities):
global all_nodes
for entity in entities:
if (isinstance(entity, ClusterCls)):
with Cluster(entity.name):
create_entities(entity.children)
elif (isinstance(entity, NodeCls)):
node = entity
node.set_ref()
all_nodes[node.name] = node
def apache(c):
with Diagram('Apache HTTP Server Pod', filename='apache', show=False, graph_attr={'pad': '0.0'}):
_apache = Apache('Apache HTTP Server')
_filebeat = Beats('Filebeat')
_metricbeat = Beats('Metricbeat')
_heartbeat = Beats('Heartbeat')
_logstash = Logstash('Logstash')
with Cluster('apache-config'):
_apache_cfg = [CM('http.conf')]
_ = _apache_cfg - _apache
with Cluster('filebeat'):
_filebeat_file = [CM('filebeat.yml')]
_ = _filebeat_file - _filebeat
with Cluster('filebeat-config'):
_filebeat_cfg = [CM('log.yml')]
_ = _filebeat_cfg - _filebeat
with Cluster('heartbeat-config'):
_heartbeat_cfg = [CM('hearbeat.yml')]
_ = _heartbeat_cfg - _heartbeat
with Cluster('heartbeat-monitor'):
_heartbeat_monitor = [CM('host.yml'), CM('logstash.yml'), CM('server.yml')]
_ = _heartbeat_monitor - _heartbeat
with Cluster('logstash-config'):
_logstash_cfg = [CM('logstash.yml'), CM('pipelines.yml'), CM('log4j2.properties')]
_ = _logstash_cfg - _logstash
with Cluster('logstash-pipeline'):
_logstash_pipe = [CM('apache.conf'), CM('system.conf')]
_ = _logstash_pipe - _logstash
with Cluster('metricbeat-config'):
_metricbeat_cfg = [CM('metricbeat.yml')]
_ = _metricbeat_cfg - _metricbeat
with Cluster('metricbeat-modules'):
_metricbeat_mod = [CM('system.yml')]
_ = _metricbeat_mod - _metricbeat
_apache >> _filebeat >> _logstash
_logstash << _metricbeat
_logstash << _heartbeat
def test_nodes_to_node_with_attributes_onedirectional(self):
with Diagram(name=os.path.join(
self.name, "nodes_to_node_with_attributes_onedirectional"),
show=False):
with Cluster():
node1 = Node("node1")
nodes = [Node("node2"), Node("node3")]
self.assertEqual(
nodes >> Edge(color="red", label="6.1") >> node1, node1)
self.assertEqual(
nodes << Edge(color="green", label="6.2") << node1, node1)
def test3():
filename = os.path.join(img_dir, sys._getframe().f_code.co_name)
with Diagram("Simple Web Service with DB Cluster", show=False, filename=filename):
dns = Route53("dns")
web = ECS("service")
with Cluster("DB Cluster"):
db_master = RDS("master")
db_master - [RDS("slave1"), RDS("slave2")]
dns >> web >> db_master
def test4():
filename = os.path.join(img_dir, sys._getframe().f_code.co_name)
with Diagram("Event Processing", show=False, filename=filename):
source = EKS("k8s source")
with Cluster("Event Flows"):
with Cluster("Event Workers"):
workers = [ECS("worker1"), ECS("worker2"), ECS("worker3")]
queue = SQS("event queue")
with Cluster("Processing"):
handlers = [Lambda("proc1"), Lambda("proc2"), Lambda("proc3")]
store = S3("events store")
dw = Redshift("analytics")
source >> workers >> queue >> handlers
handlers >> store
handlers >> dw
def draw(name: str, *, show: bool = False) -> None:
with Diagram(name):
dns = Route53()
lb = ELB()
with Cluster() as Services:
web1 = ECS()
web2 = ECS()
web3 = ECS()
with Cluster() as DBCluster:
userdb = RDS()
userdb_ro = RDS()
userdb - [userdb_ro]
memcached = ElastiCache()
dns >> lb >> Services
Services >> DBCluster
Services >> memcached
def get_common_architecture(cloud_provider, filename, container_registry,
k8s_cluster_name, k8s_engine_name):
with Diagram(
f"QHub Architecture: {cloud_provider}",
filename=f'{filename}',
show=False,
direction="TB",
):
with Cluster(cloud_provider):
gcr = ContainerRegistry(container_registry)
with Cluster(k8s_cluster_name):
kube_engine = KubernetesEngine(k8s_engine_name)
conda_pvc = PersistentVolume('Conda')
nfs_pvc = PersistentVolume('NFS')
elb = LoadBalancing('Ingress')
with Cluster('Master'):
general = Node('general')
Node('user')
Node('worker')
general << kube_engine
with Cluster("Pods"):
dask_gateway = Pod('Dask Gateway')
jupyterhub = Pod('JupyterHub')
dask_scheduler = Pod('Dask Scheduler')
nfs_server = Pod('NFS Server')
conda_store = Pod('Conda Store')
nginx = Pod('Nginx')
Pod('Cert Manager')
image_puller = Pod('Image Puller')
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
image_puller >> gcr
nfs_server >> nfs_pvc
conda_store >> conda_pvc
Helm('Helm') >> kube_engine
def get_common_architecture(cloud_provider, filename, container_registry,
k8s_cluster_name, k8s_engine_name):
with Diagram(
f"QHub Architecture: {cloud_provider}",
filename=f"{filename}",
show=False,
direction="TB",
):
with Cluster(cloud_provider):
gcr = ContainerRegistry(container_registry)
with Cluster(k8s_cluster_name):
kube_engine = KubernetesEngine(k8s_engine_name)
conda_pvc = PersistentVolume("Conda")
nfs_pvc = PersistentVolume("NFS")
elb = LoadBalancing("Ingress")
with Cluster("Master"):
general = Node("general")
Node("user")
Node("worker")
general << kube_engine
with Cluster("Pods"):
dask_gateway = Pod("Dask Gateway")
jupyterhub = Pod("JupyterHub")
dask_scheduler = Pod("Dask Scheduler")
nfs_server = Pod("NFS Server")
conda_store = Pod("Conda Store")
nginx = Pod("Nginx")
Pod("Cert Manager")
image_puller = Pod("Image Puller")
nginx >> elb
nginx >> jupyterhub
[nfs_server, conda_store, dask_gateway] << jupyterhub
[conda_store, dask_scheduler] << dask_gateway
image_puller >> gcr
nfs_server >> nfs_pvc
conda_store >> conda_pvc
Helm("Helm") >> kube_engine
def general(c):
with Diagram('General Configuration',
filename='general',
show=False,
graph_attr={'pad': '0.0'}):
_ = NS('astrid-kube')
_cb_pod = Pod('CB')
_cb_deploy = Deployment('CB')
with Cluster('Services'):
_srvs = [
Service('elasticsearch-service'),
Service('kafka-service'),
Service('cb-manager-service'),
Service('kibana')
]
with Cluster('Storage'):
_strg = PVC('elasticsearch-pv-volume') >> PV('elasticsearch-pv')
_srvs >> _cb_pod << _cb_deploy << _strg
def mysql(c, version):
with Diagram(f'MySQL Server (ver. {version}) Pod', filename=f'mysql-{version}', show=False, graph_attr={'pad': '0.0'}):
_mysql = MySQL('MySQL Server')
_metricbeat = Beats('Metricbeat')
_heartbeat = Beats('Heartbeat')
_logstash = Logstash('Logstash')
with Cluster('heartbeat-config'):
_heartbeat_cfg = [CM('hearbeat.yml')]
_ = _heartbeat_cfg - _heartbeat
with Cluster('heartbeat-monitor'):
_heartbeat_monitor = [CM('host.yml'), CM('logstash.yml'), CM('server.yml')]
_ = _heartbeat_monitor - _heartbeat
with Cluster('logstash-config'):
_logstash_cfg = [CM('logstash.yml'), CM('pipelines.yml'), CM('log4j2.properties')]
_ = _logstash_cfg - _logstash
with Cluster('logstash-pipeline'):
_logstash_pipe = [CM(f'mysql-system-{version}.conf')]
_ = _logstash_pipe - _logstash
with Cluster('metricbeat-config'):
_metricbeat_cfg = [CM('metricbeat.yml')]
_ = _metricbeat_cfg - _metricbeat
with Cluster('metricbeat-modules'):
_metricbeat_mod = [CM('system.yml')]
_ = _metricbeat_mod - _metricbeat
_mysql >> _metricbeat >> _logstash
_logstash << _heartbeat
def test_node_to_node_with_attributes(self):
with Diagram(name=os.path.join(self.name,
"node_to_node_with_attributes"),
show=False):
with Cluster():
node1 = Node("node1")
node2 = Node("node2")
self.assertEqual(
node1 << Edge(color="red", label="1.1") << node2, node2)
self.assertEqual(
node1 >> Edge(color="green", label="1.2") >> node2, node2)
self.assertEqual(
node1 << Edge(color="blue", label="1.3") >> node2, node2)
def generate_overview_diagram():
graph_attr = {
"bgcolor": "transparent"
}
with Diagram("Overview", show=False, filename="bin/overview", graph_attr=graph_attr):
with Cluster("Client"):
webapp = Angular("webapp")
with Cluster("API Services"):
status_service = Python("status_service")
task_service = Python("task_service")
worker_service = Python("worker_service")
metrics_service = Python("metrics_service")
with Cluster("Intermediate Services"):
with Cluster("Clearly Client Cluster"):
clearly_client = Python("clearly_client")
with Cluster("Backend Services"):
with Cluster("Clearly Server Cluster"):
clearly_server = Python("clearly_server")
with Cluster("External Connections"):
with Cluster("Message Broker"):
redis = Redis("redis")
with Cluster("Monitoring"):
grafana = Grafana("grafana")
prometheus = Prometheus("prometheus")
webapp >> status_service << clearly_client
webapp >> task_service << clearly_client
webapp >> worker_service << clearly_client
clearly_client >> clearly_server >> redis
metrics_service << prometheus
metrics_service >> clearly_server
