from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import Firestore
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Message Queue",
graph_attr=graph_attr,
show=False,
filename="images/message_queue"):
with Cluster("Operational Data Hub"):
with Cluster("Ingest"):
publisher_1 = Functions("Function")
publisher_2 = AppEngine("Application")
with Cluster("Consume"):
subscriber_1 = Functions("Function")
subscriber_2 = Firestore("Database")
subscriber_3 = AppEngine("Application")
with Cluster("Core"):
queue_1 = PubSub("Queue X")
queue_2 = PubSub("Queue Y")
queue_3 = PubSub("Queue Z")
publisher_1 >> queue_1 >> [subscriber_1, subscriber_2]
publisher_1 >> queue_3 >> subscriber_3
publisher_2 >> queue_2 >> [subscriber_1, subscriber_3]
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import BigQuery, Dataflow, PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import BigTable
from diagrams.gcp.iot import IotCore
from diagrams.gcp.storage import GCS
with Diagram("Message Collecting", show=False):
pubsub = PubSub("pubsub")
with Cluster("Source of Data"):
[IotCore("core1"), IotCore("core2"), IotCore("core3")] >> pubsub
with Cluster("Targets"):
with Cluster("Data Flow"):
flow = Dataflow("data flow")
with Cluster("Data Lake"):
flow >> [BigQuery("bq"), GCS("storage")]
with Cluster("Event Driven"):
with Cluster("Processing"):
flow >> AppEngine("engine") >> BigTable("bigtable")
with Cluster("Serverless"):
flow >> Functions("func") >> AppEngine("appengine")
pubsub >> flow
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import BigQuery, Dataflow, PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import BigTable
from diagrams.gcp.storage import GCS
from diagrams.programming.framework import React
with Diagram("Data project3:Bitcoin"):
pubsub = PubSub("Pub Sub")
with Cluster("Source data"):
yfsd = Functions("yahoo finance")
yfsd >> pubsub
with Cluster("Artificial Intelligence - Bitcoin"):
flow = Functions("ETL process")
with Cluster("Data base"):
db = BigQuery("Big Query")
flow >> db
predict = Functions("Calculate Prediction")
with Cluster("Update model"):
CTF = Functions("Control Training")
train = Functions("Training")
with Cluster("Web"):
webpage = React("React.App")
pubsub >> flow
#db >> train
db >> predict
db >> CTF #
}
with Diagram("Pub/Sub", graph_attr=graph_attr, direction="TB", show=False, filename="images/pubsub"):
with Cluster("Publishers"):
publisher_a = AppEngine("Publisher A")
publisher_b = AppEngine("Publisher B")
publisher_c = Functions("Publisher C")
with Cluster("Subscribers"):
subscriber_x = AppEngine("Subscriber X")
subscriber_y = Functions("Subscriber Y")
subscriber_z = Functions("Subscriber Z")
with Cluster("Cloud Pub/Sub"):
with Cluster("Topic A"):
topic_a = PubSub("Topic A")
subscription_ay = PubSub("Subscription AY")
subscription_az = PubSub("Subscription AZ")
with Cluster("Topic B"):
topic_b = PubSub("Topic B")
subscription_bx = PubSub("Subscription BX")
with Cluster("Topic C"):
topic_c = PubSub("Topic C")
subscription_cx = PubSub("Subscription CX")
publisher_a >> topic_a
topic_a >> subscription_ay >> subscriber_y
topic_a >> subscription_az >> subscriber_z
publisher_b >> topic_b >> subscription_bx >> subscriber_x
from diagrams.onprem.compute import Server
from diagrams.onprem.client import Client
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Customer support mailbox",
graph_attr=graph_attr,
show=False,
filename="images/mailbox"):
server_1 = Server("Mail server")
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic"):
pubsub_1 = PubSub("Subscription")
with Cluster("Ingest Project"):
schedule_1 = Scheduler("Cloud Scheduler")
function_1 = Functions("EWS Mail Ingest")
storage_1 = Storage("GCS Bucket")
with Cluster("Consume Project"):
function_2 = Functions("Consume")
firestore_1 = Firestore("Database")
appengine_1 = AppEngine("API")
schedule_1 - Edge(label="Trigger", style="dotted") - function_1
function_1 >> Edge(label="REST - GET", color="black") >> server_1
function_1 >> Edge(label="Publish", color="orange") >> pubsub_1
function_1 >> storage_1 << Edge(label="GET", color="orange") << function_2
cqrs_1_2 = Storage("Command Model")
webshop_1 = Client("Webshop")
firestore_1_1 >> Edge(color="orange") >> cqrs_1_1 >> Edge(color="orange") >> \
Edge(label="Presentation updates", color="orange") >> webshop_1
webshop_1 >> Edge(label="User changes") >> cqrs_1_2 >> firestore_1_1
with Diagram("ODH CQRS integration",
graph_attr=graph_attr,
show=False,
filename="images/cqrs_odh_model"):
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic Z"):
pubsub_2_1 = PubSub("Subscription ZX")
pubsub_2_2 = PubSub("Subscription ZY")
with Cluster("Consume Project X"):
consume_2_1 = Functions("Consume X")
firestore_2_1 = Firestore("Database X")
with Cluster("CQRS Model"):
cqrs_2_1 = Storage("Query Model")
consume_2_1 >> firestore_2_1 >> cqrs_2_1
with Cluster("Consume Project Y"):
consume_2_2 = Functions("Consume Y")
firestore_2_2 = Firestore("Database Y")
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Current Stock",
graph_attr=graph_attr,
show=False,
filename="images/current_stock"):
server_1 = Server("Third-party API")
server_2 = Server("Third-party API")
server_3 = Server("Third-party API")
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic X"):
pubsub_1 = PubSub("Subscription XA")
with Cluster("Pub/Sub Topic Y"):
pubsub_2 = PubSub("Subscription YA")
with Cluster("Ingest Project"):
function_1 = Functions("Restingest")
function_2 = Functions("Produce delta event")
storage_1 = Storage("GCS Bucket")
function_1 >> storage_1
storage_1 - Edge(label="Bucket Trigger",
style="dashed") - function_2
with Cluster("Consume Project"):
function_3 = Functions("Consume")
from diagrams.onprem.compute import Server
from diagrams.onprem.client import Client
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Posting third-party license data",
graph_attr=graph_attr,
show=False,
filename="images/renewal_licenses_posting"):
webshop_1 = Client("Webshop")
server_1_1 = Server("Abode API")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic"):
pubsub_1_1 = PubSub("Subscription")
with Cluster("Ingest Project"):
function_1_1 = Functions("Restingest")
function_1_2 = Functions("Produce delta event")
storage_1_1 = Storage("GCS Bucket")
function_1_1 >> storage_1_1
storage_1_1 - Edge(label="Bucket Trigger",
style="dashed") - function_1_2 >> Edge(
label="Publish") >> pubsub_1_1
with Cluster("Consume Project"):
function_1_3 = Functions("Consume")
webshop_1 >> Edge(label="POST", color="black") >> function_1_1
public_registries = Custom("Public Registries",
"../../resources/diag-registry-icon.png")
with Cluster("GCP account (sysdig)", graph_attr={"bgcolor": "lightblue"}):
sds = Custom("Sysdig Secure", "../../resources/diag-sysdig-icon.png")
bench = General("Cloud Bench")
sds >> Edge(label="schedule on rand rand * * *") >> bench
with Cluster("GCP organization project", graph_attr={"bgcolor": "pink"}):
ccProjectSink = Custom("\nLog Router \n Sink",
"../../resources/sink.png")
orgBenchRole = Iam("Cloud Bench Role")
with Cluster("Secure for Cloud (children project)"):
ccBenchRole = Iam("Cloud Bench Role")
ccPubSub = PubSub("CC PubSub Topic")
ccEventarc = Code("CloudRun\nEventarc Trigger")
ccCloudRun = Run("Cloud Connector")
bucket = GCS("Bucket\nCC Config")
keys = KMS("Sysdig \n Secure Keys")
ccCloudRun << Edge(style="dashed") << keys
bucket << Edge(style="dashed",
label="Get CC \n config file") << ccCloudRun
ccEventarc >> ccCloudRun
ccEventarc << ccPubSub
ccProjectSink >> ccPubSub
gcrPubSub = PubSub("GCR PubSub Topic\n(gcr named)")
gcrSubscription = Code("GCR PubSub\nSubscription")
csCloudBuild = Build("Triggered\n Cloud Builds")
:authors
JP at 19/04/20
"""
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import PubSub
from diagrams.gcp.compute import Functions
from diagrams.gcp.storage import GCS
from diagrams.gcp.analytics import BigQuery
from diagrams.gcp.database import Firestore
from diagrams.gcp.devtools import Scheduler
with Diagram("Strava Leaderboard Architecture Diagram ", show=True):
source = Functions("generate grid")
with Cluster("Data Pipeline"):
gird_queue = PubSub("grid queue")
credential = Firestore("credentials store")
store = GCS("raw JSON")
with Cluster("Extract-Load"):
with Cluster("scheduler"):
scheduler = Scheduler("scheduler")
schedule_queue = PubSub("schedule queue")
extract_load = Functions("worker")
staging = BigQuery("BigQuery staging dataset")
with Cluster("Transform"):
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Data Science",
graph_attr=graph_attr,
show=False,
filename="images/data_science"):
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Ingest Project"):
function_1 = Functions("Ingest")
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic X"):
pubsub_1_1 = PubSub("Subscription XA")
with Cluster("Pub/Sub Topic Z"):
pubsub_2_1 = PubSub("Subscription ZA")
with Cluster("Consume Project"):
dataflow_1 = Dataflow("Dataflow")
with Cluster("Analyze"):
bigquery_1 = Bigquery("BigQuery")
aiplatform_1 = AIPlatform("AI Platform")
firestore_2 = Firestore("Database")
dataflow_2 = Dataflow("Backfill/reprocess")
pubsub_1_1 >> dataflow_1
dataflow_1 >> [bigquery_1, aiplatform_1, firestore_2] >> dataflow_2