def init_app(self, local_dynamodb=None, prefix="", skip=False):
name_template = prefix + "{table_name}"
if local_dynamodb is not None:
self._init_local(local_dynamodb, name_template)
else:
self.engine = Engine(table_name_template=name_template)
self.engine.bind(self.Model, skip_table_setup=skip)
def engine(session, dynamodb, dynamodbstreams):
# HACK: These clients won't be used. We're going to replace the session immediately.
engine = Engine(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams)
# Toss the clients above and hook up the mock session
engine.session = session
engine.bind(BaseModel)
return engine
class DynamoDb:
def __init__(self):
self._local_dynamodb = None
self.Model = BaseModel
self.engine = None
def _init_local(self,
local_dynamodb="http://127.0.0.1:4444",
name_template="{table_name}"):
dynamodb = boto3.client("dynamodb", endpoint_url=local_dynamodb)
dynamodbstreams = boto3.client("dynamodbstreams",
endpoint_url=local_dynamodb)
self.engine = Engine(dynamodb=dynamodb,
dynamodbstreams=dynamodbstreams,
table_name_template=name_template)
client = patch_engine(self.engine)
client.mock_ttl["MyTableName"] = True
client.mock_backups["MyTableName"] = False
def init_app(self, local_dynamodb=None, prefix="", skip=False):
name_template = prefix + "{table_name}"
if local_dynamodb is not None:
self._init_local(local_dynamodb, name_template)
else:
self.engine = Engine(table_name_template=name_template)
self.engine.bind(self.Model, skip_table_setup=skip)
def _init_local(self,
local_dynamodb="http://127.0.0.1:4444",
name_template="{table_name}"):
dynamodb = boto3.client("dynamodb", endpoint_url=local_dynamodb)
dynamodbstreams = boto3.client("dynamodbstreams",
endpoint_url=local_dynamodb)
self.engine = Engine(dynamodb=dynamodb,
dynamodbstreams=dynamodbstreams,
table_name_template=name_template)
client = patch_engine(self.engine)
client.mock_ttl["MyTableName"] = True
client.mock_backups["MyTableName"] = False
def engine_for_region(region, table_name_template="{table_name}"):
dynamodb = boto3.client("dynamodb", region_name=region)
dynamodbstreams = boto3.client("dynamodbstreams", region_name=region)
return Engine(
dynamodb=dynamodb,
dynamodbstreams=dynamodbstreams,
table_name_template=table_name_template
)
def engine(dynamodb, dynamodbstreams, request):
engine = Engine(dynamodb=dynamodb,
dynamodbstreams=dynamodbstreams,
table_name_template="{table_name}" +
request.config.getoption("--nonce"))
yield engine
# This collects all subclasses of BaseModel and are not abstract. We are trying to delete any data in
# dynamodb-local between unit tests so we don't step on each other's toes.
concrete = set(
filter(lambda m: not m.Meta.abstract, walk_subclasses(BaseModel)))
for model in concrete:
# we can run into a situation where the class was created, but not bound in the engine (or table created), so
# we only try. As the dynamodb-local process is only running in memory this isn't too much of a problem.
try:
objs = list(engine.scan(model))
if objs:
engine.delete(*objs)
except BloopException:
pass
def push(engine: Engine, id: str) -> Tuple[int, int]:
"""Returns (status, position) tuple.
next_position should be a guess; always guess low, or misordering can occur.
Position will be the reserved queue position, even if the user already existed."""
# -1. Non-authoritative check for existing user.
# This doesn't guarantee the user doesn't exist, it just saves a lot of empty queue positions
# when a malicious actor tries to enqueue themselves many times.
# (Mitigation factor depends on )
try:
existing = engine.query(Person, key=Person.id == id).first()
# If we don't hit an exception, person already exists
return 409, existing.position
except ConstraintViolation:
# Probably doesn't exist, keep going
pass
# 0. Reserve a row in QueueEntry with the next queue position.
# Don't insert the id until we confirm that this user
# hasn't already been enqueued.
entry = QueueEntry(position=0, enqueued_at=pendulum.now(), served_at=None)
while True:
try:
engine.save(entry, condition=QueueEntry.NOT_EXIST)
except ConstraintViolation:
# TODO | can be much faster:
# TODO | - use a binary walk from 0 until we hit a missing,
# TODO | - then binary walk back to an existing,
# TODO | - then use try save with increment of 1
entry.position += 1
else:
break
# 1. Now that we've reserved a row, try to build the id -> position mapping.
# Note that a failure here means the user is already queued, so we abort.
# Can't roll back the QueueEntry though, since another caller may have already
# advanced past us.
person = Person(id=id, position=entry.position)
try:
engine.save(person, condition=Person.NOT_ENQUEUED)
except ConstraintViolation:
return 409, entry.position
# 2. Success! Finally, push the id into the QueueEntry.
# It's ok if this fails, because we can always rebuild the id from the Person table.
entry.id = id
try:
engine.save(entry)
except BloopException:
# Can't be sure the entry is saved, but it's fine because we'll
# be able to rebuild from the Person
return 404, entry.position
return 200, entry.position
def serve(engine: Engine, id: str) -> int:
"""200 success, 404 missing row, 409 already served"""
# 0. Find the QueueEntry by Person
person = Person(id=id)
try:
engine.load(person)
except MissingObjects:
return 404
entry = QueueEntry(position=person.position)
try:
engine.load(entry)
except MissingObjects:
return 404
# 1. Update the time the entry was served at.
# If the entry was already served, don't change the time.
entry.served_at = pendulum.now()
try:
engine.save(entry, condition=QueueEntry.served_at.is_(None))
except ConstraintViolation:
# Already served, no change
return 409
else:
return 200
def engine(dynamodb, dynamodbstreams):
engine = Engine(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams)
yield engine
engine.delete(*engine.scan(User))
if len(args) >= 2:
stage = args[1]
config_filename = 'config.' + stage + '.json'
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
config_filepath = os.path.join(parent_dir, config_filename)
with open(config_filepath, 'r') as fp:
config = json.load(fp)
region = config['REGION']
try:
endpoint = args[2]
client = boto3.client('dynamodb', region_name=region, endpoint_url=endpoint)
engine = patch_engine(Engine(dynamodb=client))
except IndexError:
client = boto3.client('dynamodb', region_name=region)
engine = Engine(dynamodb=client)
resp = client.list_tables()
tables = resp['TableNames']
for model in models:
model.Meta.table_name = model.Meta.table_name.format(STAGE=stage)
print('Running dynamodb tables creation script '
'in Region: {REGION}'.format(REGION=region))
print("Dynamodb Tables: ")
for model in models:
if model.Meta.table_name not in tables:
stage = args[1]
os.environ['STAGE'] = stage
config_filename = 'config.' + stage + '.json'
parent_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
config_filepath = os.path.join(parent_dir, config_filename)
with open(config_filepath, 'r') as fp:
config = json.load(fp)
region = config['REGION']
client = boto3.client('dynamodb', region_name=region)
engine = Engine(dynamodb=client)
model = Deployment
# pull all items from deployment table
table = model.Meta.table_name
print("Updating brewoptix-deployment with new schema")
print("Getting all items in table")
all_items = client.scan(TableName=table)
# delete existing table
print("Delete table")
resp = client.delete_table(TableName=table)
for _ in range(10):
import json
from os import environ
from uuid import uuid4, UUID
import boto3
from bloop import Engine, ConstraintViolation
from todoapi.models.todo import TodoItem
if environ.get("AWS_SAM_LOCAL") == "true":
dynamodb = boto3.client('dynamodb', endpoint_url="http://dynamodb:8000")
db = Engine(dynamodb=dynamodb)
db.bind(TodoItem)
else:
dynamodb = boto3.client('dynamodb')
db = Engine(dynamodb=dynamodb)
db.bind(TodoItem, skip_table_setup=True
) # we can skip the table setup because CloudFormation will do it.
def get(event, context):
if event['pathParameters']:
todo_item = db.query(TodoItem,
key=TodoItem.uuid == UUID(
event['pathParameters']['todo_id'])).one()
body = todo_item.as_dict
else:
todos = db.scan(TodoItem)
body = {"todos": [todo_item.as_dict for todo_item in todos]}
return {"body": json.dumps(body), "statusCode": 200}
from . import aws
from . import exc
import uuid
from bloop import Engine, UUID, Boolean, Column, ConstraintViolation
engine = Engine(session=aws.session)
def default(kwargs, key, value):
kwargs[key] = kwargs.get(key, value)
class Blob(engine.model):
class Meta:
table_name = "blobapy-blob"
key_name = Column(UUID, hash_key=True, name='k')
admin_key = Column(UUID, name='a')
deleted = Column(Boolean, name='d')
NOT_EXISTS = key_name.is_(None)
def __init__(self, **kwargs):
default(kwargs, "deleted", False)
super().__init__(**kwargs)
@classmethod
def unique(cls):
with exc.on_botocore("Failed to generate unique key_name"):
retries = 5
while retries:
obj = cls(key_name=uuid.uuid4(), admin_key=uuid.uuid4())
class Tweet(BaseModel):
class Meta:
write_units = 10
account = Column(UUID, hash_key=True)
id = Column(String, range_key=True)
content = Column(String)
date = Column(DateTime)
favorites = Column(Integer)
by_date = GlobalSecondaryIndex(hash_key='date', projection='keys')
engine = Engine()
engine.bind(BaseModel)
# ================================================
# Usage
# ================================================
account = Account(id=uuid.uuid4(), name='@garybernhardt', email='REDACTED')
tweet = Tweet(account=account.id,
id='616102582239399936',
content='today, I wrote a type validator in Python, as you do',
favorites=9,
date=datetime.now(timezone.utc))
engine.save(account, tweet)
if __name__ == "__main__":
if __name__ == '__main__':
import sys
import json
import os
args = sys.argv
if len(args) >= 2:
stage = args[1]
config_filename = 'config.' + stage + '.json'
parent_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
config_filepath = os.path.join(parent_dir, config_filename)
try:
with open(config_filepath, 'r') as fp:
config = json.load(fp)
except FileNotFoundError:
print("Cannot find config file: {0}".format(config_filename))
sys.exit(1)
region = config['REGION']
client = boto3.client('dynamodb', region_name=region)
engine = Engine(dynamodb=client)
print("Creating deployment table in Dynamodb: ")
print('Creating table: ', Deployment.Meta.table_name)
engine.bind(Deployment)
import arrow
import uuid
from bloop import (Engine, Column, Integer, DateTime, UUID,
GlobalSecondaryIndex, String, new_base)
engine = Engine()
Base = new_base()
class Account(Base):
class Meta:
read_units = 5
write_units = 2
id = Column(UUID, hash_key=True)
name = Column(String)
email = Column(String)
by_email = GlobalSecondaryIndex(hash_key='email',
projection='keys_only',
write_units=1,
read_units=5)
class Tweet(Base):
class Meta:
write_units = 10
account = Column(UUID, hash_key=True)
id = Column(String, range_key=True)
content = Column(String)
date = Column(DateTime)
favorites = Column(Integer)
'Rating': Number,
'Updated': DateTime,
'Description': Map(**{
'Title': String,
'Body': String
}),
'Sellers': Set(Integer)
})
class Item(BaseModel):
id = Column(UUID, hash_key=True)
data = Column(Product)
engine = Engine()
engine.bind(BaseModel)
# ================================================
# Usage
# ================================================
item = Item(id=uuid.uuid4())
item.data = {
'Name': 'item-name',
'Rating': decimal.Decimal(str(random.random())),
'Updated': datetime.now(timezone.utc),
'Description': {
'Title': 'item-title',
'Body': 'item-body',
},
import arrow
import boto3
import decimal
import random
import uuid
from bloop import (Engine, Column, DateTime, Integer,
UUID, String, Map, TypedMap, Float)
# ================================================
# Model setup
# ================================================
session = boto3.session.Session(profile_name="test-user-bloop")
engine = Engine(session=session)
Product = Map(**{
'Name': String,
'Rating': Float,
'Updated': DateTime('US/Pacific'),
'Description': Map(**{
'Title': String,
'Body': String
}),
'Sellers': TypedMap(Integer)
})
class Item(engine.model):
id = Column(UUID, hash_key=True)
def bind_all(engine: Engine) -> None:
engine.bind(Person)
engine.bind(QueueEntry)
import arrow
import boto3
import uuid
from bloop import (Engine, Column, Integer, DateTime, UUID,
GlobalSecondaryIndex, String)
# ================================================
# Model setup
# ================================================
session = boto3.session.Session(profile_name="test-user-bloop")
engine = Engine(session=session)
class Account(engine.model):
class Meta:
read_units = 5
write_units = 2
id = Column(UUID, hash_key=True)
name = Column(String)
email = Column(String)
by_email = GlobalSecondaryIndex(
hash_key='email', projection='keys_only',
write_units=1, read_units=5)
class Tweet(engine.model):
class Meta:
write_units = 10
account = Column(UUID, hash_key=True)
from bloop import BaseModel, Column, String, Integer, Engine
from bloop.exceptions import ConstraintViolation
class NumberStore(BaseModel):
key = Column(String, hash_key=True)
value = Column(Integer)
engine = Engine(table_name_template="my-memory-{table_name}")
engine.bind(NumberStore)
def get(key):
try:
return engine.query(NumberStore,
key=NumberStore.key == key).one().value
except ConstraintViolation:
return None
def set(key, value):
update = NumberStore(key=key, value=value)
engine.save(update)
class Paste(SortByVersion, BaseModel):
class Meta:
ttl = {"column": "not_after"}
not_after = Column(Timestamp, default=new_expiry)
bucket = Column(String, dynamo_name="b")
key = Column(String, dynamo_name="k")
class UserImage(SortByVersion, BaseModel):
jpg = Column(Binary)
engine = Engine()
engine.bind(BaseModel)
def s3_upload(content: str) -> (str, str):
# TODO persist in s3
return "bucket-id", "key-id"
def b64sha256(content: str) -> str:
hash = hashlib.sha256(content.encode())
return base64.b64encode(hash.digest()).decode()
def new_paste(content: str) -> str:
id = b64sha256(content)
def engine(dynamodb, dynamodbstreams):
return Engine(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams)
product_id = Column(String, range_key=True, dynamo_name="pid")
sku = Column(String, dynamo_name="sku")
variation = Column(String, dynamo_name="v")
weight = Column(String, dynamo_name="w")
name = Column(String, dynamo_name="n")
class VendorImports(BaseModel):
class Meta:
table_name = "teascraper.VendorImports"
vendor_id = Column(String, hash_key=True, dynamo_name="id")
last_run_at = Column(DateTime, dynamo_name="lr")
engine = Engine()
models = [Product, VendorImports, ScrapedData]
# super hack to handle table setup
# in read-only prod environment
for model in models:
try:
engine.bind(model)
except BloopException:
print("Skipping table setup for " + model.__name__)
engine.bind(model, skip_table_setup=True)
def make_product_id(vendor: str, name: str, weight: str):
name = name.lower().replace(" ", "-").replace(".", "_")
product_name = vendor + "." + name + "." + weight
return product_name