def s3_events_to_db(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates S3 events to DB diagrams """
output = f"{output_path}/event_processing"
with Diagram(
"Event Processing",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
with Cluster("Serverless"):
source = S3("S3 events")
with Cluster("Event Flows"):
with Cluster("Concurrent Processing"):
handlers = [
Lambda("Python"),
Lambda("Python"),
Lambda("Python")
]
dw = Dynamodb("S3 metadata index")
source >> handlers
handlers >> dw
return f"{output}.{output_format}"
def backend(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates backend diagrams """
output = f"{output_path}/backend"
with Diagram(
"Backend",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
with Cluster("Serverless"):
apigw = APIGateway("API Gateway")
with Cluster("Concurrent Processing"):
handlers = [
Lambda("API (FastAPI)"),
Lambda("API (FastAPI)"),
Lambda("API (FastAPI)"),
]
db = Dynamodb("data store")
apigw >> handlers >> db >> handlers >> apigw
return f"{output}.{output_format}"
def apigw_dynamodb_lambda():
stack_objective = "apigw-dynamodb-lambda"
with Diagram(stack_objective,
outformat="png",
filename=f"{stack_objective}/pics/arch",
show=False):
apigw = APIGateway("/task")
dynamodb = Dynamodb("DynamoDB")
apigw >> Edge(
label="POST /example/update") >> Lambda("update status") >> Edge(
label="update item") >> dynamodb
apigw >> Edge(label="POST /example") >> Lambda("producer") >> Edge(
label="put item") >> dynamodb
apigw >> Edge(label="GET /example") >> Lambda("consumer") >> Edge(
label="read all item") >> dynamodb
def process_media(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates media processing diagrams """
output = f"{output_path}/media_processing"
with Diagram(
"Media Processing",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
cli = Python("CLI")
with Cluster("Serverless"):
source = SQS("tasks queue")
with Cluster("Concurrent Processing"):
handlers = Lambda("convert image\nencode video")
with Cluster("Lambda layers"):
[Layer("ImageMagic"), Layer("ffmpeg")]
src = S3("input\nmedia files")
dest = S3("output\nmedia files")
cli >> source >> handlers << src
handlers >> dest
return f"{output}.{output_format}"
def apigw_lambda():
stack_objective = "apigw-lambda"
with Diagram(stack_objective,
outformat="png",
filename=f"{stack_objective}/pics/arch",
show=False):
APIGateway("APIGateway") >> Edge(label="integration") >> Lambda("task")
def test4():
filename = os.path.join(img_dir, sys._getframe().f_code.co_name)
with Diagram("Event Processing", show=False, filename=filename):
source = EKS("k8s source")
with Cluster("Event Flows"):
with Cluster("Event Workers"):
workers = [ECS("worker1"), ECS("worker2"), ECS("worker3")]
queue = SQS("event queue")
with Cluster("Processing"):
handlers = [Lambda("proc1"), Lambda("proc2"), Lambda("proc3")]
store = S3("events store")
dw = Redshift("analytics")
source >> workers >> queue >> handlers
handlers >> store
handlers >> dw
def apigw_dynamodb_sfn_with_heavytask():
stack_objective = "apigw-dynamodb-sfn-with-heavytask"
with Diagram(stack_objective,
outformat="png",
filename=f"{stack_objective}/pics/arch",
show=False):
sqs = SQS("SQS")
apigw = APIGateway("/task") >> Lambda("integration") >> [
sqs, Dynamodb("DynamoDB")
]
timer_lambda = Lambda("timer lambda")
sqs << Edge(label="dequeue") << timer_lambda << Cloudwatch("cron")
with Cluster(label="StepFunctions", direction="TB"):
sfn_start = SFn_TASK("update DynamoDB\nset `running`")
sfn_start \
>> Lambda("Some Task") \
>> [SFn_TASK("update DynamoDB\nset `success`"), SFn_TASK("update DynamoDB\nset `failure`")]
# invoke sfn from Lambda
timer_lambda >> sfn_start
def main():
with Diagram('New SFTP Process') as diag:
user = Windows('User')
with Cluster('Repay-CDE-PROD-SFTP', direction='TB'):
sftp_service = TransferForSftp('SFTP')
s3_sftp_bucket = S3('channels_sftp_transfer')
sftp_service - s3_sftp_bucket
aws_sftp_group = [sftp_service, s3_sftp_bucket]
user >> Edge(label='Upload a BLF into SFTP') >> sftp_service
with Cluster('repay-cde-prod-channels', direction='TB'):
lambda_blf_copy = Lambda('s3_copy_lambda')
s3_blf_processor = S3('blf_processor bucket')
sns_blf_uploaded_to_s3 = SNS('SNS - blf_uploaded_to_s3')
redis = ElastiCache('(redis)')
elasticsearch = ES('1 index per BLF')
lambda_blf_copy >> s3_blf_processor
s3_blf_processor >> sns_blf_uploaded_to_s3
cde_group = [
lambda_blf_copy, s3_blf_processor, sns_blf_uploaded_to_s3,
redis, elasticsearch
]
with Cluster('repay-cde-prod-k8s', direction='TB'):
k8s_api_pod = Pod('Channels API\n/api/v1/blf_upload_sns')
k8s_blf_processor_job = Job('process-blf-k8s')
k8s_api_pod >> Edge(
label='Create job if no BLF lock exists for org/filename'
) >> k8s_blf_processor_job
k8s_group = [k8s_api_pod, k8s_blf_processor_job]
# TODO - MAKE SURE TO HIGHLIGHT THE USE OF INVISIBLE EDGES
s3_sftp_bucket >> Edge(style='invis') >> cde_group
redis >> Edge(style='invis') >> k8s_group
elasticsearch >> Edge(style='invis') >> k8s_group
k8s_blf_processor_job << Edge(
label='Download file from s3') << s3_blf_processor
s3_sftp_bucket >> Edge(
label='S3 notification Object Created') >> lambda_blf_copy
sns_blf_uploaded_to_s3 >> Edge(
label='HTTP request with BLF file name') >> k8s_api_pod
k8s_api_pod >> Edge(
label='Create BLF lock - 5 minute expiration') >> redis
k8s_blf_processor_job >> Edge(
label='Delete BLF lock when done') >> redis
k8s_blf_processor_job >> Edge(label='Create ES index') >> elasticsearch
def frontend(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates frontend diagrams """
output = f"{output_path}/frontend"
with Diagram(
"Frontend",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
client = Client("client")
with Cluster("Serverless"):
with Cluster("UI"):
with Cluster("cache"):
web_cdn = CloudFront("CDN\nUI")
with Cluster("static"):
web_host = S3("web")
react = React("app")
with Cluster("Static Assets"):
with Cluster("cache"):
assets_cdn = CloudFront("CDN\nassets")
assets_apigw = APIGateway("API Gateway")
assets = S3("assets\nimages")
with Cluster("media processing"):
assets_gen = Lambda("generate image")
layers = Layer("layer\nImageMagick")
web_cdn << react << web_host
assets_cdn << assets_apigw << assets_gen << layers << assets
client - Edge(color="orange") << assets_cdn
client - Edge(color="orange") << web_cdn
assets_apigw >> assets_gen
assets_cdn >> assets_apigw
return f"{output}.{output_format}"
def main():
with Diagram("Recipeify Architecture", direction='LR'):
with Cluster("Github"):
ci = Github("Actions CI/CD")
Github("Open Source Repo") >> ci
with Cluster("Cloudfare"):
clf = Cloudflare("CDN\n DDoS Protection")
with Cluster("AWS", direction='LR'):
with Cluster("VPC"):
elb = ELB("Elastic Load Balancer")
eb = EB("Elastic Beanstalk")
es = ES("Elasticsearch")
lmda = Lambda("Lambda")
ecr = ECR("Container Registry")
with Cluster("3rd Party SaaS", direction='LR'):
with Cluster("MongoDB Atlas"):
mongo = MongoDB("Users DB")
with Cluster("Recombee"):
rec = Recombee("Recommendation\n Engine")
with Cluster("Auth0"):
auth = Auth0("Oauth2 Integrator")
with Cluster("Cloudinary"):
cl = Cloudinary("Image Caching\n and Proccessing")
auth >> Edge(style="invis") >> mongo >> Edge(
style="invis") >> cl >> Edge(style="invis") >> rec
eb >> auth
eb >> mongo
eb >> cl
eb >> rec
eb << ecr << ci
es << lmda
clf >> elb >> eb >> es
def cli(
graph_attr: dict = GRAPH_ATTR,
output_format: str = OUTPUT_FORMAT,
output_path: str = OUTPUT_PATH,
) -> str:
""" Generates CLI diagrams """
output = f"{output_path}/cli"
with Diagram(
"CLI",
show=False,
outformat=output_format,
# graph_attr=graph_attr,
filename=output,
):
cli = Python("CLI")
with Cluster("targets"):
with Cluster("local resources"):
data = Storage("build objects\nassets")
layers = Layer("build\nLambda layers")
server = Server("media\nlocal build")
with Cluster("cloud resources"):
db = Dynamodb("create DB\nseed DB")
s3 = S3("sync data")
lbd = Lambda("resize images\nencode video")
sqs = SQS("queue\nmedia generate\ntasks")
cli >> data >> cli
cli >> s3
cli >> db
cli >> lbd
cli >> sqs
cli >> layers
cli >> server
return f"{output}.{output_format}"
class Extended(Diagram):
def render(self):
self.dot.render(format=self.outformat, view=False, quiet=True)
if self.outformat == "svg":
path = pathlib.Path(f"{self.filename}.{self.outformat}")
t = ET.parse(str(path))
fix_image_links(t)
logger.info("fix links %s", self.filename)
t.write(str(path))
# TODO: support self.show
with Extended(
"architecture",
show=True, # not supported yet
direction="LR",
outformat="svg",
filename="architecture.embed",
):
amplify = Amplify("AWS Amplify")
api_gateway = APIGateway("Amazon API Gateway")
aws_lambda = Lambda("AWS lambda")
iam = IAM("AWS Identity and Access Management")
dynamo_db = Dynamodb("Amazon DynamoDB")
amplify >> api_gateway >> aws_lambda >> dynamo_db
iam >> aws_lambda
# print("@", to_base64.cache_info())
from diagrams import Diagram
from diagrams.aws.compute import Lambda
from diagrams.aws.database import DynamodbTable
from diagrams.aws.integration import SNS
from diagrams.aws.network import APIGateway
with Diagram("User Profile", show=False, direction="TB"):
userprofile = Lambda("user profile")
# topic = SNS("user profile")
APIGateway("user profile api") >> userprofile >> DynamodbTable("user profiles") >> userprofile
# userprofile >> topic
from diagrams import Cluster, Diagram
from diagrams.aws.compute import Lambda, Fargate
from diagrams.aws.database import Dynamodb
from diagrams.aws.integration import SQS, SNS, SF
from diagrams.aws.network import APIGateway
from diagrams.aws.storage import S3
from diagrams.aws.analytics import Kinesis, KinesisDataFirehose
with Diagram("SNS Integration", direction="LR", show=False):
apig = APIGateway("Producer (http post)")
topic = SNS("Topic")
with Cluster("Consumers"):
consumers_l = [
Lambda("Consumer"),
Lambda("Consumer"),
Lambda("Consumer")
]
apig >> topic
consumers_l >> topic
curated = S3("CuratedData")
submissions_crawler = GlueCrawlers("submissions crawler")
curated_crawler = Glue("ETL")
ctas = Athena("train/eval split")
catalog = GlueDataCatalog("data catalog")
notebooks = SagemakerNotebook("Build Model")
job = SagemakerTrainingJob("Train Model")
model = SagemakerModel("Fitted Model")
with Cluster("Inference"):
endpointLambda = Lambda("call endpoint")
with Cluster("Multi AZ endpoints") as az:
endpoints = [
Endpoint("us-east-1a"),
Endpoint("us-east-1b"),
Endpoint("us-east-1c"),
]
published = S3("Monitor data")
monitor_sched = EMR("model monitor")
source >> submissions >> submissions_crawler >> curated_crawler >> curated
submissions >> catalog
iot >> inference >> source
curated >> ctas >> [catalog, job]
notebooks >> job >> model
"fontsize": "45"
}
node_attr = {
"esep": "+20",
"fontsize": "10"
}
with Diagram("On Demand Athena Reports", show=True, direction='LR',
graph_attr=graph_attr, node_attr =node_attr):
_ddb = DDB('DynamoDB Table \nReports Definitions (queries)')
_lambda = Lambda('Lambda Publisher \nPOST {report_id: 100, client_id:1}')
_lambda << Edge() >> _ddb
_api = APIGateway('Rest API')
_client = Client('Client API Request Report ID=100')
_client >> _api
_api >> Edge(color="darkgreen") >> _lambda
with Cluster("Reports Queue "):
_sqs =SQS('Amazon SQS\nReport Request Queue')
_lambda >> _sqs
_lambda2 = Lambda('Lambda Subscriber \nProcess Queue Messages \n (start query)')
_sqs >> _lambda2
from diagrams import Cluster, Diagram
from diagrams.aws.compute import Lambda
with Diagram("The Secret Formula", show=False, direction="TB"):
who = Lambda("Who?")
where = Lambda("Where?")
bait = Lambda("Bait?")
result = Lambda("Result?")
with Cluster("Who?", direction="TB"):
whoQuestions = [
Lambda("Who is your dream client?"),
Lambda("Who do I actually want to work with?"),
Lambda("What are they passionate about?"),
Lambda("What are their goals, dreams, and desires?")
]
with Cluster("Where?"):
whereQuestions = [
Lambda("Where can you find your dream clients?"),
Lambda("What groups are they a part of?"),
Lambda("What interests do they have?"),
Lambda("Where do they hang out online?")
]
with Cluster("Bait?"):
baitQuestions = [
Lambda("What bait will you use to attract your dream clients?"),
Lambda("Is your bait a physical book?"),
Lambda("Is your bait a CD?"),
Lambda("Is your bait an audio file?")
]
from diagrams import Diagram, Edge
from diagrams.aws.compute import Lambda
from diagrams.aws.engagement import SES
from diagrams.aws.integration import SNS
with Diagram("Calendar Mail Send", show=False, direction="TB"):
topic = SNS("calendar topic")
ses = SES("simple email service")
topic >> Edge(
label="published calendar") >> Lambda("calendar mail send") >> Edge(
label="multipart email") >> ses
# diagram.py
from diagrams import Cluster, Diagram, Edge
from diagrams.aws.compute import Lambda
from diagrams.aws.network import APIGateway
graph_attr = {"bgcolor": "transparent"}
with Diagram("CSV to Heartbeat YAML",
show=False,
outformat="png",
filename="overview",
graph_attr=graph_attr):
lambda_converter = Lambda("Converter")
lambda_retention = Lambda("Lambda Retention policy")
gw = APIGateway("API Gateway")
gw >> lambda_converter
#!/usr/bin/env python
from diagrams import Diagram
from diagrams.aws.compute import Lambda
from diagrams.aws.ml import Personalize
from diagrams.aws.analytics import KinesisDataStreams, KinesisDataFirehose, Athena, Quicksight, Glue
from diagrams.onprem.client import Client
from diagrams.aws.storage import S3
with Diagram('イベントストリーミング'):
# インスタンス化によってノードを作成
# ノードにラベルを付与でき、\nを入れることでラベルの改行も可能
stream = KinesisDataStreams('Kinesis\nData Streams')
s3 = S3('S3')
athena = Athena('Athena')
# 定義したノードを始点とした流れを作成
# 変数に代入せずとも、ノードは作成可能
Client() >> stream >> Lambda('Lambda') >> Personalize('Personalize\nEventTracker')
stream >> KinesisDataFirehose('Kinesis\nData Firehose') >> s3
s3 - athena >> Quicksight('QuickSight') << Client()
s3 >> Glue('Glue') >> athena
from diagrams import Diagram
from diagrams.aws.compute import Lambda
from diagrams.aws.integration import SNS
from diagrams.aws.network import APIGateway
with Diagram("Calendar User Agent", show=False, direction="TB"):
cua = Lambda("calendar user agent")
# APIGateway("calendar user agent api") >> cua
topic = SNS("calendar topic")
events = APIGateway("calendar events")
tasks = APIGateway("calendar tasks")
journal = APIGateway("calendar journal")
availability = APIGateway("calendar availability")
topic >> cua >> topic
events >> cua >> events
tasks >> cua >> tasks
journal >> cua >> journal
availability >> cua >> availability
from diagrams import Cluster, Diagram
from diagrams.aws.compute import Lambda, Fargate
from diagrams.aws.database import Dynamodb
from diagrams.aws.integration import Eventbridge
from diagrams.aws.network import APIGateway
from diagrams.aws.storage import S3
with Diagram("Message Broker Refactored", show=False):
with Cluster("Event Processors"):
with Cluster("System C", direction="TB"):
apig = APIGateway("webhook")
handler = Lambda("handler")
with Cluster("System D", direction="TB"):
processor = Lambda("handler")
ddb = Dynamodb("database")
stream_listener = Lambda("processor")
broker = Eventbridge("message broker")
with Cluster("System B", direction="TB"):
event_handler = Lambda("handler")
bucket = S3("S3")
file_processor = Lambda("processor")
apig >> handler >> broker >> processor
processor >> ddb >> stream_listener >> broker
from diagrams.programming.framework import Vue, Django
from diagrams.aws.compute import Lambda
from diagrams.aws.engagement import Connect
from diagrams.aws.analytics import Kinesis
from diagrams.aws.database import Dynamodb
from pathlib import Path
with Diagram(
"CC3 AWS Connect High Level Architecture",
filename="architecture",
outformat="png",
show=False,
direction="BT",
):
connect = Connect("Connect")
connectLambda = Lambda("awsConnect")
web = Vue("CCU3 Web")
api = Django("CCU3 Api")
users = [Users("Users"), Users("Agents")]
with Cluster("CCU3 awsconnect"):
gateway = APIGateway("Websocket Gateway")
database = Dynamodb("ConnectClientsDB")
with Cluster("Websocket Handler"):
with Cluster("Connection"):
connectionHandlers = [Lambda("$connect" + str(i)) for i in range(3)]
gateway - connectionHandlers
connectionHandlers >> database
with Cluster("AWS"):
security = Cognito("Cognito")
gateway = APIGateway("Gateway")
route = Route53("Route53")
db = DDB("DynamoDB")
email_service = SES("SES")
monitoring = Cloudwatch("AWS CloudWatch ")
firewall = WAF("AWS WAF")
identity = IAM("AWS IAM")
with Cluster("CDN"):
cdn = S3("S3") >> CF("CloudFront CDN")
with Cluster("Functions") as xyz:
func_send_mail = Lambda("Send Email")
func_store_data = Lambda("Store Data")
functions = [func_send_mail, func_store_data]
gateway >> Edge() << functions
functions >> Edge() << identity
func_send_mail >> Edge() >> email_service >> users
func_store_data - Edge() - db
cdn >> Edge() << route
# Monitoring
log_connection = Edge(color="darkpink", style="dotted")
monitoring >> log_connection << gateway
monitoring >> log_connection << [func_send_mail, func_store_data]
monitoring >> log_connection << firewall
from diagrams import Diagram
from diagrams.aws.compute import Lambda
with Diagram("AWS Lambda Layer", show=False, direction="TB"):
Lambda("lambda layer")
with Diagram(None, filename="aws-cross-account-pipeline", show=False):
developers = Users("Developers")
with Cluster("Developer Account"):
source_code = Codecommit("CodeCommit")
source_code << Edge(label="merge pr") << developers
with Cluster("Shared Services Account"):
with Cluster("Pipeline"):
pipeline = Codepipeline("CodePipeline")
build = Codebuild("Codebuild")
artifacts = S3("Build Artifacts")
source_code >> Edge(label="trigger") >> pipeline
developers >> Edge(label="manual approval") >> pipeline
pipeline >> build >> Edge(label="yaml file") >> artifacts
with Cluster("Test Workload Account"):
test_stack = Cloudformation("CloudFormation")
test_function = Lambda("Lambda")
test_api = APIGateway("API Gateway")
pipeline >> test_stack
test_api >> test_function
with Cluster("Prod Workload Account"):
prod_stack = Cloudformation("CloudFormation")
prod_function = Lambda("Lambda")
prod_api = APIGateway("API Gateway")
pipeline >> prod_stack
prod_api >> prod_function
from diagrams import Cluster, Diagram
from diagrams.aws.compute import ECS, EKS, Lambda
from diagrams.aws.database import Redshift
from diagrams.aws.integration import SQS
from diagrams.aws.storage import S3
with Diagram("Event Processing", show=False):
source = EKS("k8s source")
with Cluster("Event Flows"):
with Cluster("Event Workers"):
workers = [ECS("worker1"), ECS("worker2"), ECS("worker3")]
queue = SQS("event queue")
with Cluster("Processing"):
handlers = [Lambda("proc1"), Lambda("proc2"), Lambda("proc3")]
store = S3("events store")
dw = Redshift("analytics")
source >> workers >> queue >> handlers
handlers >> store
handlers >> dw
from diagrams import Diagram, Cluster
from diagrams.aws.storage import S3
from diagrams.aws.compute import Lambda
from diagrams.aws.integration import SNS, Eventbridge
from diagrams.aws.management import Cloudwatch
from diagrams.onprem.queue import ActiveMQ
with Diagram("Alerting Workflow", show=True):
with Cluster('main account'):
topic = SNS('SNS Topic')
with Cluster('Lambda'):
l = Lambda('processor')
topic >> l
S3('lambda source') - l
cl = Cloudwatch('Cloudwatch')
l >> cl
event = Eventbridge('Cloudwatch\nevent rule')
cl >> event
with Cluster('Event Bus'):
event_bus = ActiveMQ('bus')
event >> event_bus
from diagrams import Diagram
from diagrams.aws.network import APIGateway
from diagrams.aws.compute import Lambda
from diagrams.aws.database import Aurora
with Diagram("Lambda Invocation", show=False):
api_gw = APIGateway("API Gateway")
my_lambda = Lambda("Lambda")
my_db = Aurora("Aurora Database")
api_gw >> my_lambda >> my_db
from diagrams.aws.compute import Lambda
from diagrams.aws.storage import S3
from diagrams.aws.network import APIGateway
from diagrams.aws.database import DynamodbTable
from diagrams.aws.security import IdentityAndAccessManagementIam
from diagrams.aws.devtools import Codebuild
from diagrams.aws.devtools import Codecommit
from diagrams.aws.devtools import Codedeploy
from diagrams.aws.devtools import Codepipeline
from diagrams.aws.management import Cloudformation
from diagrams.aws.devtools import CommandLineInterface
with Diagram("Serverless Web Apps", show=False, direction="TB"):
with Cluster("CloudFormation"):
cloudformation = Cloudformation("Stack")
cloudformation >> IdentityAndAccessManagementIam("IAM") >> Codecommit(
"CodeCommit") >> Codebuild("CodeBuild") >> S3(
"S3") >> Codepipeline("CodePipeline")
with Cluster("CodePipeline"):
codepipeline = Codepipeline("Pipeline")
codepipeline >> Codecommit("CodeCommit") >> Codebuild(
"CodeBuild") >> Cloudformation("CloudFormation")
with Cluster("Serverless Application Model"):
sam = Cloudformation("SAM Template")
sam >> APIGateway("API Gateway") >> Lambda("Lambda") >> DynamodbTable(
"DynamoDB")
cloudformation >> codepipeline >> sam