from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import BigQuery, Dataflow, PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import BigTable
from diagrams.gcp.iot import IotCore
from diagrams.gcp.storage import GCS
with Diagram("Message Collecting", show=False):
pubsub = PubSub("pubsub")
with Cluster("Source of Data"):
[IotCore("core1"), IotCore("core2"), IotCore("core3")] >> pubsub
with Cluster("Targets"):
with Cluster("Data Flow"):
flow = Dataflow("data flow")
with Cluster("Data Lake"):
flow >> [BigQuery("bq"), GCS("storage")]
with Cluster("Event Driven"):
with Cluster("Processing"):
flow >> AppEngine("engine") >> BigTable("bigtable")
with Cluster("Serverless"):
flow >> Functions("func") >> AppEngine("appengine")
pubsub >> flow
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import PubSub
from diagrams.gcp.compute import AppEngine, Functions
graph_attr = {
"bgcolor": "transparent",
"pad": "0"
}
with Diagram("Pub/Sub", graph_attr=graph_attr, direction="TB", show=False, filename="images/pubsub"):
with Cluster("Publishers"):
publisher_a = AppEngine("Publisher A")
publisher_b = AppEngine("Publisher B")
publisher_c = Functions("Publisher C")
with Cluster("Subscribers"):
subscriber_x = AppEngine("Subscriber X")
subscriber_y = Functions("Subscriber Y")
subscriber_z = Functions("Subscriber Z")
with Cluster("Cloud Pub/Sub"):
with Cluster("Topic A"):
topic_a = PubSub("Topic A")
subscription_ay = PubSub("Subscription AY")
subscription_az = PubSub("Subscription AZ")
with Cluster("Topic B"):
topic_b = PubSub("Topic B")
subscription_bx = PubSub("Subscription BX")
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import Firestore
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Message Queue",
graph_attr=graph_attr,
show=False,
filename="images/message_queue"):
with Cluster("Operational Data Hub"):
with Cluster("Ingest"):
publisher_1 = Functions("Function")
publisher_2 = AppEngine("Application")
with Cluster("Consume"):
subscriber_1 = Functions("Function")
subscriber_2 = Firestore("Database")
subscriber_3 = AppEngine("Application")
with Cluster("Core"):
queue_1 = PubSub("Queue X")
queue_2 = PubSub("Queue Y")
queue_3 = PubSub("Queue Z")
publisher_1 >> queue_1 >> [subscriber_1, subscriber_2]
publisher_1 >> queue_3 >> subscriber_3
publisher_2 >> queue_2 >> [subscriber_1, subscriber_3]
firestore_1_1 >> Edge(color="orange") >> cqrs_1_1 >> Edge(color="orange") >> \
Edge(label="Presentation updates", color="orange") >> webshop_1
webshop_1 >> Edge(label="User changes") >> cqrs_1_2 >> firestore_1_1
with Diagram("ODH CQRS integration",
graph_attr=graph_attr,
show=False,
filename="images/cqrs_odh_model"):
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic Z"):
pubsub_2_1 = PubSub("Subscription ZX")
pubsub_2_2 = PubSub("Subscription ZY")
with Cluster("Consume Project X"):
consume_2_1 = Functions("Consume X")
firestore_2_1 = Firestore("Database X")
with Cluster("CQRS Model"):
cqrs_2_1 = Storage("Query Model")
consume_2_1 >> firestore_2_1 >> cqrs_2_1
with Cluster("Consume Project Y"):
consume_2_2 = Functions("Consume Y")
firestore_2_2 = Firestore("Database Y")
with Cluster("CQRS Model"):
cqrs_2_2 = Storage("Query Model")
consume_2_2 >> firestore_2_2 >> cqrs_2_2
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import BigQuery, Dataflow, PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import BigTable
from diagrams.gcp.storage import GCS
from diagrams.programming.framework import React
with Diagram("Data project3:Bitcoin"):
pubsub = PubSub("Pub Sub")
with Cluster("Source data"):
yfsd = Functions("yahoo finance")
yfsd >> pubsub
with Cluster("Artificial Intelligence - Bitcoin"):
flow = Functions("ETL process")
with Cluster("Data base"):
db = BigQuery("Big Query")
flow >> db
predict = Functions("Calculate Prediction")
with Cluster("Update model"):
CTF = Functions("Control Training")
train = Functions("Training")
with Cluster("Web"):
webpage = React("React.App")
pubsub >> flow
#db >> train
db >> predict
db >> CTF #
with Diagram("Customer support mailbox",
graph_attr=graph_attr,
show=False,
filename="images/mailbox"):
server_1 = Server("Mail server")
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic"):
pubsub_1 = PubSub("Subscription")
with Cluster("Ingest Project"):
schedule_1 = Scheduler("Cloud Scheduler")
function_1 = Functions("EWS Mail Ingest")
storage_1 = Storage("GCS Bucket")
with Cluster("Consume Project"):
function_2 = Functions("Consume")
firestore_1 = Firestore("Database")
appengine_1 = AppEngine("API")
schedule_1 - Edge(label="Trigger", style="dotted") - function_1
function_1 >> Edge(label="REST - GET", color="black") >> server_1
function_1 >> Edge(label="Publish", color="orange") >> pubsub_1
function_1 >> storage_1 << Edge(label="GET", color="orange") << function_2
pubsub_1 >> Edge(label="Subscribe", color="orange") >> function_2 >> Edge(
color="orange") >> firestore_1
firestore_1 << Edge(label="GET") << appengine_1 << Edge(
label="REST - GET", color="black") << webshop_1
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import BigQuery, Dataflow, PubSub
from diagrams.gcp.compute import AppEngine, Functions
from diagrams.gcp.database import BigTable
from diagrams.gcp.iot import IotCore
from diagrams.gcp.storage import GCS
with Diagram("Media Monitoring Storage Architecture", show=False) as med_diag:
pubsub = PubSub("pubsub")
flow = Dataflow("DataFlow")
with Cluster("Data Collection"):
[
Functions("RSS Feed Webhook"),
Functions("Twitter Webhook"),
Functions("Press Release")
] >> pubsub >> flow
with Cluster("Storage"):
with Cluster("Data Lake"):
flow >> [BigQuery("BigQuery"), GCS("Storage")]
with Cluster("Event Driven"):
with Cluster("Processing"):
flow >> AppEngine("GAE") >> BigTable("BigTable")
with Cluster("Serverless"):
flow >> Functions("Function") >> AppEngine("AppEngine")
pubsub >> flow
filename="images/current_stock"):
server_1 = Server("Third-party API")
server_2 = Server("Third-party API")
server_3 = Server("Third-party API")
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic X"):
pubsub_1 = PubSub("Subscription XA")
with Cluster("Pub/Sub Topic Y"):
pubsub_2 = PubSub("Subscription YA")
with Cluster("Ingest Project"):
function_1 = Functions("Restingest")
function_2 = Functions("Produce delta event")
storage_1 = Storage("GCS Bucket")
function_1 >> storage_1
storage_1 - Edge(label="Bucket Trigger",
style="dashed") - function_2
with Cluster("Consume Project"):
function_3 = Functions("Consume")
server_1 >> Edge(label="REST - POST", color="blue") >> function_1
server_2 >> Edge(label="REST - POST", color="orange") >> function_1
server_3 >> Edge(label="REST - POST", color="orange") >> function_1
function_2 >> Edge(label="Publish", color="blue") >> pubsub_1 >> Edge(
label="Subscribe", color="blue") >> function_3
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Legacy ERP system",
graph_attr=graph_attr,
show=False,
filename="images/legacy_erp_system"):
server_1 = Server("NaaB IV")
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic"):
pubsub_1 = PubSub("Subscription")
with Cluster("Ingest Project"):
function_1 = Functions("Restingest")
function_2 = Functions("Produce delta event")
scheduler_1 = Scheduler("Cloud Scheduler")
storage_1 = Storage("GCS Bucket")
scheduler_1 - Edge(label="Trigger", style="dotted") - function_1
server_1 << Edge(label="GET",
color="black") << function_1 >> storage_1
storage_1 >> Edge(label="Bucket Trigger") >> function_2 >> Edge(
label="Publish") >> pubsub_1
with Cluster("Consume Project"):
function_3 = Functions("Event sourcing consumer")
database_1 = Firestore("Database")
appengine_1 = AppEngine("API")
"""Create architecture diagram for data pipeline.
:authors
JP at 19/04/20
"""
from diagrams import Cluster, Diagram
from diagrams.gcp.analytics import PubSub
from diagrams.gcp.compute import Functions
from diagrams.gcp.storage import GCS
from diagrams.gcp.analytics import BigQuery
from diagrams.gcp.database import Firestore
from diagrams.gcp.devtools import Scheduler
with Diagram("Strava Leaderboard Architecture Diagram ", show=True):
source = Functions("generate grid")
with Cluster("Data Pipeline"):
gird_queue = PubSub("grid queue")
credential = Firestore("credentials store")
store = GCS("raw JSON")
with Cluster("Extract-Load"):
with Cluster("scheduler"):
scheduler = Scheduler("scheduler")
schedule_queue = PubSub("schedule queue")
extract_load = Functions("worker")
staging = BigQuery("BigQuery staging dataset")
from diagrams.onprem.vcs import Github
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Available Data",
graph_attr=graph_attr,
show=False,
filename="images/available_data"):
github_1 = Github("GitHub repo")
with Cluster("Operational Data Hub Platform"):
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic"):
pubsub_1 = PubSub("Subscription")
with Cluster("Ingest Project"):
function_1 = Build("Cloud Build")
with Cluster("Consume Project"):
function_2 = Functions("Consume catalog")
with Cluster("CKAN"):
computeengine_1 = ComputeEngine("CKAN VM")
sql_1 = SQL("PostgreSQL")
github_1 >> Edge(
label="Build Trigger",
color="black") >> function_1 >> Edge(label="Publish") >> pubsub_1
pubsub_1 >> Edge(label="Subscribe") >> function_2 >> sql_1
sql_1 >> Edge(label="SSL") >> computeengine_1
from diagrams.gcp.database import Firestore
from diagrams.gcp.ml import AIPlatform
from diagrams.onprem.client import Client
graph_attr = {"bgcolor": "transparent", "pad": "0"}
with Diagram("Data Science",
graph_attr=graph_attr,
show=False,
filename="images/data_science"):
webshop_1 = Client("Webshop")
with Cluster("Operational Data Hub Platform"):
with Cluster("Ingest Project"):
function_1 = Functions("Ingest")
with Cluster("Operational Data Hub"):
with Cluster("Pub/Sub Topic X"):
pubsub_1_1 = PubSub("Subscription XA")
with Cluster("Pub/Sub Topic Z"):
pubsub_2_1 = PubSub("Subscription ZA")
with Cluster("Consume Project"):
dataflow_1 = Dataflow("Dataflow")
with Cluster("Analyze"):
bigquery_1 = Bigquery("BigQuery")
aiplatform_1 = AIPlatform("AI Platform")
firestore_2 = Firestore("Database")