def test_list_tables(self):
"""
Connection.list_tables
"""
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), LIST_TABLE_DATA
conn = Connection(self.region)
conn.list_tables(exclusive_start_table_name='Thread')
self.assertEqual(req.call_args[1], {'exclusive_start_table_name': 'Thread'})
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), LIST_TABLE_DATA
conn = Connection(self.region)
conn.list_tables(limit=3)
self.assertEqual(req.call_args[1], {'limit': 3})
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), LIST_TABLE_DATA
conn = Connection(self.region)
conn.list_tables()
self.assertEqual(req.call_args[1], {})
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), None
conn = Connection(self.region)
self.assertRaises(TableError, conn.list_tables)
def remove_from_db(self, uid: str) -> bool:
try:
conn = Connection(host=self.model.Meta.host,
region=self.model.Meta.region)
conn.delete_item(self.model.Meta.table_name,
hash_key=uid,
range_key=None) # Note that range key is optional
return True
except DeleteError as e:
logger.error("failed to remove user from db", uid=uid)
return False
def test_delete_table(self):
"""
Connection.delete_table
"""
params = {'table_name': 'ci-table'}
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn = Connection(self.region)
conn.delete_table(self.test_table_name)
kwargs = req.call_args[1]
self.assertEqual(kwargs, params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), None
conn = Connection(self.region)
self.assertRaises(TableError, conn.delete_table, self.test_table_name)
def connect(cls):
cls.connection = Connection(host=cls.DDB_LOCAL_HOST,
region=cls.DDB_LOCAL_REGION)
try:
cls.connection.list_tables()
except (TableError, ConnectionError) as e:
print(e)
raise EnvironmentError(
"DynamoDB Local does not appear to be running on localhost, port {}! Cannot run tests!"
)
def test_signal_exception_post_signal(mock_uuid, mock_req):
pre_recorded = []
UUID = '123-abc'
def record_pre_dynamodb_send(sender, operation_name, table_name, req_uuid):
pre_recorded.append((operation_name, table_name, req_uuid))
def record_post_dynamodb_send(sender, operation_name, table_name, req_uuid):
raise ValueError()
pre_dynamodb_send.connect(record_pre_dynamodb_send)
post_dynamodb_send.connect(record_post_dynamodb_send)
try:
mock_uuid.uuid4.return_value = UUID
mock_req.return_value = {'TableDescription': {'TableName': 'table', 'TableStatus': 'Creating'}}
c = Connection()
c.dispatch('CreateTable', {'TableName': 'MyTable'})
assert ('CreateTable', 'MyTable', UUID) == pre_recorded[0]
finally:
pre_dynamodb_send.disconnect(record_pre_dynamodb_send)
post_dynamodb_send.disconnect(record_post_dynamodb_send)
def test_signal_exception_post_signal(mock_uuid, mock_req):
pre_recorded = []
UUID = '123-abc'
def record_pre_dynamodb_send(sender, operation_name, table_name, req_uuid):
pre_recorded.append((operation_name, table_name, req_uuid))
def record_post_dynamodb_send(sender, operation_name, table_name, req_uuid):
raise ValueError()
pre_dynamodb_send.connect(record_pre_dynamodb_send)
post_dynamodb_send.connect(record_post_dynamodb_send)
try:
mock_uuid.uuid4.return_value = UUID
mock_req.return_value = {'TableDescription': {'TableName': 'table', 'TableStatus': 'Creating'}}
c = Connection()
c.dispatch('CreateTable', {'TableName': 'MyTable'})
assert ('CreateTable', 'MyTable', UUID) == pre_recorded[0]
finally:
pre_dynamodb_send.disconnect(record_pre_dynamodb_send)
post_dynamodb_send.disconnect(record_post_dynamodb_send)
def handler(event, context):
# Validation
request = Request(**loads(event.get("body", {})))
# Business logic, running safely under a transaction
with TransactWrite(connection=Connection(),
client_request_token=generate_uuid()) as transaction:
transaction.save(
Store(
name=request.name,
delay=request.delay,
working_since=request.working_since.strftime(
"%Y-%m-%d %H:%M:%S"),
))
for employee_name in request.employees:
transaction.save(Employee(name=employee_name))
def test_get_item(self):
"""
Connection.get_item
"""
conn = Connection(self.region)
table_name = 'Thread'
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), GET_ITEM_DATA
item = conn.get_item(table_name, "Amazon DynamoDB", "How do I update multiple items?")
self.assertEqual(item, GET_ITEM_DATA)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), None
self.assertRaises(
GetError,
conn.get_item,
table_name,
"Amazon DynamoDB",
"How do I update multiple items?"
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), GET_ITEM_DATA
conn.get_item(
table_name,
"Amazon DynamoDB",
"How do I update multiple items?",
attributes_to_get=['ForumName']
)
params = {
'return_consumed_capacity': 'TOTAL',
'attributes_to_get': ['ForumName'],
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'consistent_read': False,
'table_name': 'Thread'
}
self.assertEqual(req.call_args[1], params)
def test_commit(self, mocker):
connection = Connection()
mock_connection_transact_get = mocker.patch.object(
connection, 'transact_get_items')
with patch(PATCH_METHOD) as req:
req.return_value = MOCK_TABLE_DESCRIPTOR
with TransactGet(connection=connection) as t:
t.get(MockModel, 1, 2)
mock_connection_transact_get.assert_called_once_with(
get_items=[{
'Key': {
'MockHash': {
'N': '1'
},
'MockRange': {
'N': '2'
}
},
'TableName': 'mock'
}],
return_consumed_capacity=None)
def test_describe_table(self):
"""
Connection.describe_table
"""
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn = Connection(self.region)
conn.describe_table(self.test_table_name)
self.assertEqual(req.call_args[1], {'table_name': 'ci-table'})
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), DESCRIBE_TABLE_DATA
conn = Connection(self.region)
table = conn.describe_table(self.test_table_name)
self.assertIsNone(table)
with patch(PATCH_METHOD) as req:
req.return_value = HttpUnavailable(), None
conn = Connection(self.region)
self.assertRaises(TableError, conn.describe_table, self.test_table_name)
def update_capacity(self, dynamo_host, dynamo_region, table_root, logger, config):
"""
Manage the DynamoDB tables:
- Create if the tables don't exist
- Update the read/write capacity if they do exist and there is a conflict between required and current
:param dynamo_host: optional host, for local testing
:param dynamo_region: aws region for the table
:param table_root: prefix for table_name (e.g. flyby)
:param logger:
:return:
"""
models = [BackendModel, ServiceModel, TargetGroupModel, ResolverModel]
for model in models:
if dynamo_host:
model.Meta.host = dynamo_host
conn = Connection(host=dynamo_host)
else:
conn = Connection(region=dynamo_region)
default_table_name = model.Meta.table_name
if not model.Meta.table_name.startswith(table_root):
model.Meta.table_name = "{0}-{1}".format(table_root, model.Meta.table_name)
model.Meta.region = dynamo_region
if not model.exists():
logger.info("Creating {} table".format(model.Meta.table_name))
read_capacity_units = self.return_capacity(default_table_name, config)['read_capacity_units']
write_capacity_units = self.return_capacity(default_table_name, config)['write_capacity_units']
model.create_table(read_capacity_units=read_capacity_units,
write_capacity_units=write_capacity_units,
wait=True
)
else:
table_name = model.Meta.table_name
table_capacity = self.capacity_check(default_table_name, table_name, conn, config)
if not table_capacity['result'] and table_capacity['decreases'] < 4:
conn.update_table(
table_name=model.Meta.table_name, read_capacity_units=table_capacity['read'],
write_capacity_units=table_capacity['write']
)
logger.info("Updating {} table read/write capacity".format(model.Meta.table_name))
elif not table_capacity['result'] and table_capacity['write'] >= 4:
logger.error("Unable to decrease capacity on {} table".format(model.Meta.table_name))
def test_query(self):
"""
Connection.query
"""
conn = Connection()
table_name = 'Thread'
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(table_name)
self.assertRaises(
ValueError,
conn.query,
table_name,
"FooForum",
return_consumed_capacity='TOTAL',
key_conditions={'ForumName': {'ComparisonOperator': 'BAD_OPERATOR', 'AttributeValueList': ['thread']}}
)
self.assertRaises(
ValueError,
conn.query,
table_name,
"FooForum",
return_consumed_capacity='TOTAL',
select='BAD_VALUE',
key_conditions={'ForumName': {'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': ['thread']}}
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
QueryError,
conn.query,
table_name,
"FooForum",
scan_index_forward=True,
return_consumed_capacity='TOTAL',
select='ALL_ATTRIBUTES',
key_conditions={'ForumName': {'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': ['thread']}}
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.query(
table_name,
"FooForum",
scan_index_forward=True,
return_consumed_capacity='TOTAL',
select='ALL_ATTRIBUTES',
key_conditions={'ForumName': {'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': ['thread']}}
)
params = {
'scan_index_forward': True,
'select': 'ALL_ATTRIBUTES',
'return_consumed_capacity': 'TOTAL',
'key_conditions': {
'ForumName': {
'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': [{
'S': 'thread'
}]
}
},
'table_name': 'Thread'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.query(
table_name,
"FooForum",
key_conditions={'ForumName': {'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': ['thread']}}
)
params = {
'return_consumed_capacity': 'TOTAL',
'key_conditions': {
'ForumName': {
'ComparisonOperator': 'BEGINS_WITH', 'AttributeValueList': [{
'S': 'thread'
}]
}
},
'table_name': 'Thread'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.query(
table_name,
"FooForum",
limit=1,
index_name='LastPostIndex',
attributes_to_get=['ForumName'],
exclusive_start_key="FooForum",
consistent_read=True
)
params = {
'limit': 1,
'return_consumed_capacity': 'TOTAL',
'consistent_read': True,
'exclusive_start_key': {
'ForumName': {
'S': 'FooForum'
}
},
'index_name': 'LastPostIndex',
'attributes_to_get': ['ForumName'],
'key_conditions': {
'ForumName': {
'ComparisonOperator': 'EQ', 'AttributeValueList': [{
'S': 'FooForum'
}]
}
},
'table_name': 'Thread'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.query(
table_name,
"FooForum",
select='ALL_ATTRIBUTES',
exclusive_start_key="FooForum"
)
params = {
'return_consumed_capacity': 'TOTAL',
'exclusive_start_key': {
'ForumName': {
'S': 'FooForum'
}
},
'key_conditions': {
'ForumName': {
'ComparisonOperator': 'EQ', 'AttributeValueList': [{
'S': 'FooForum'
}]
}
},
'table_name': 'Thread',
'select': 'ALL_ATTRIBUTES'
}
self.assertEqual(req.call_args[1], params)
def test_batch_get_item(self):
"""
Connection.batch_get_item
"""
items = []
conn = Connection()
table_name = 'Thread'
for i in range(10):
items.append(
{"ForumName": "FooForum", "Subject": "thread-{0}".format(i)}
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
GetError,
conn.batch_get_item,
table_name,
items,
consistent_read=True,
return_consumed_capacity='TOTAL',
attributes_to_get=['ForumName']
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_get_item(
table_name,
items,
consistent_read=True,
return_consumed_capacity='TOTAL',
attributes_to_get=['ForumName']
)
params = {
'return_consumed_capacity': 'TOTAL',
'request_items': {
'Thread': {
'consistent_read': True,
'attributes_to_get': ['ForumName'],
'Keys': [
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}
]
}
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_get_item(
table_name,
items
)
params = {
'return_consumed_capacity': 'TOTAL',
'request_items': {
'Thread': {
'Keys': [
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}},
{'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}
]
}
}
}
self.assertEqual(req.call_args[1], params)
# AWS Lambda
if (os.environ.get("LAMBDA_TASK_ROOT",
None) is not None # Running inside Lambda
and
bool(os.getenv("AWS_SAM_LOCAL")) is False # But not on SAM local
):
return None
# SAM Local
# `dynamodb` is the service name inside
# docker-compose.yml
return "http://dynamodb:8000"
host = get_nosql_database_url()
connection = (Connection(host=get_nosql_database_url())
if host else Connection(region=os.environ["AWS_REGION"]))
logger.debug(f"Session Registry created for {connection}")
class Base(Model):
"""Base relational class.
Use this class to add functionality
to PynamoDB Base class.
"""
created_at = UTCDateTimeAttribute(default=get_time_now)
updated_at = UTCDateTimeAttribute(default=get_time_now)
cls = DiscriminatorAttribute()
def test_create_table(self):
"""
Connection.create_table
"""
conn = Connection(self.region)
kwargs = {
'read_capacity_units': 1,
'write_capacity_units': 1,
}
self.assertRaises(ValueError, conn.create_table, self.test_table_name, **kwargs)
kwargs['attribute_definitions'] = [
{
'attribute_name': 'key1',
'attribute_type': 'S'
},
{
'attribute_name': 'key2',
'attribute_type': 'S'
}
]
self.assertRaises(ValueError, conn.create_table, self.test_table_name, **kwargs)
kwargs['key_schema'] = [
{
'attribute_name': 'key1',
'key_type': 'hash'
},
{
'attribute_name': 'key2',
'key_type': 'range'
}
]
params = {
'table_name': 'ci-table',
'provisioned_throughput': {
'WriteCapacityUnits': 1,
'ReadCapacityUnits': 1
},
'attribute_definitions': [
{
'AttributeType': 'S',
'AttributeName': 'key1'
},
{
'AttributeType': 'S',
'AttributeName': 'key2'
}
],
'key_schema': [
{
'KeyType': 'HASH',
'AttributeName': 'key1'
},
{
'KeyType': 'RANGE',
'AttributeName': 'key2'
}
]
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), None
self.assertRaises(TableError, conn.create_table, self.test_table_name, **kwargs)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn.create_table(
self.test_table_name,
**kwargs
)
self.assertEqual(req.call_args[1], params)
kwargs['global_secondary_indexes'] = [
{
'index_name': 'alt-index',
'key_schema': [
{
'KeyType': 'HASH',
'AttributeName': 'AltKey'
}
],
'projection': {
'ProjectionType': 'KEYS_ONLY'
},
'provisioned_throughput': {
'ReadCapacityUnits': 1,
'WriteCapacityUnits': 1,
},
}
]
params['global_secondary_indexes'] = [{'IndexName': 'alt-index', 'Projection': {'ProjectionType': 'KEYS_ONLY'},
'KeySchema': [{'AttributeName': 'AltKey', 'KeyType': 'HASH'}],
'ProvisionedThroughput': {'ReadCapacityUnits': 1,
'WriteCapacityUnits': 1}}]
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn.create_table(
self.test_table_name,
**kwargs
)
self.assertEqual(req.call_args[1], params)
del(kwargs['global_secondary_indexes'])
del(params['global_secondary_indexes'])
kwargs['local_secondary_indexes'] = [
{
'index_name': 'alt-index',
'projection': {
'ProjectionType': 'KEYS_ONLY'
},
'key_schema': [
{
'AttributeName': 'AltKey', 'KeyType': 'HASH'
}
],
'provisioned_throughput': {
'ReadCapacityUnits': 1,
'WriteCapacityUnits': 1
}
}
]
params['local_secondary_indexes'] = [
{
'Projection': {
'ProjectionType': 'KEYS_ONLY'
},
'KeySchema': [
{
'KeyType': 'HASH',
'AttributeName': 'AltKey'
}
],
'IndexName': 'alt-index'
}
]
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn.create_table(
self.test_table_name,
**kwargs
)
self.assertEqual(req.call_args[1], params)
import os
import boto3
import urllib
import pyexcel as pe
from pynamodb.connection import Connection
# database permit model
from models import PermitsModel
# Our created utils
from utils import (write_records, create_permit, adjust_capacity)
s3 = boto3.client('s3')
Permits = PermitsModel('eu-central-1',
'https://dynamodb.eu-central-1.amazonaws.com')
conn = Connection(region='eu-central-1',
host='https://dynamodb.eu-central-1.amazonaws.com')
def lambda_handler(event, context):
# get table name from CloudFormation template
dynamo_table = os.environ['TableName']
# set temp file
# '/tmp' is directory to write to inside Lambda function container
report_file = '/tmp/report.xlsx'
# Get bucket and key from PUT event
bucket = event['Records'][0]['s3']['bucket']['name'].encode('utf8')
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
def test_update_table(self):
"""
Connection.update_table
"""
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn = Connection(self.region)
params = {
'provisioned_throughput': {
'WriteCapacityUnits': 2,
'ReadCapacityUnits': 2
},
'table_name': 'ci-table'
}
conn.update_table(
self.test_table_name,
read_capacity_units=2,
write_capacity_units=2
)
self.assertEqual(req.call_args[1], params)
self.assertRaises(ValueError, conn.update_table, self.test_table_name, read_capacity_units=2)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), None
conn = Connection(self.region)
self.assertRaises(
TableError,
conn.update_table,
self.test_table_name,
read_capacity_units=2,
write_capacity_units=2)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), None
conn = Connection(self.region)
global_secondary_index_updates = [
{
"index_name": "foo-index",
"read_capacity_units": 2,
"write_capacity_units": 2
}
]
params = {
'table_name': 'ci-table',
'provisioned_throughput': {
'ReadCapacityUnits': 2,
'WriteCapacityUnits': 2,
},
'global_secondary_index_updates': [
{
'Update': {
'IndexName': 'foo-index',
'ProvisionedThroughput': {
'ReadCapacityUnits': 2,
'WriteCapacityUnits': 2,
}
}
}
]
}
conn.update_table(
self.test_table_name,
read_capacity_units=2,
write_capacity_units=2,
global_secondary_index_updates=global_secondary_index_updates
)
self.assertEqual(req.call_args[1], params)
def test_batch_write_item(self):
"""
Connection.batch_write_item
"""
items = []
conn = Connection()
table_name = 'Thread'
for i in range(10):
items.append(
{"ForumName": "FooForum", "Subject": "thread-{0}".format(i)}
)
self.assertRaises(
ValueError,
conn.batch_write_item,
table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_write_item(
table_name,
put_items=items,
return_item_collection_metrics='SIZE',
return_consumed_capacity='TOTAL'
)
params = {
'return_consumed_capacity': 'TOTAL',
'return_item_collection_metrics': 'SIZE',
'request_items': {
'Thread': [
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}}}
]
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_write_item(
table_name,
put_items=items
)
params = {
'return_consumed_capacity': 'TOTAL',
'request_items': {
'Thread': [
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}}}},
{'PutRequest': {'Item': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}}}
]
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
PutError,
conn.batch_write_item,
table_name,
delete_items=items
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_write_item(
table_name,
delete_items=items
)
params = {
'return_consumed_capacity': 'TOTAL',
'request_items': {
'Thread': [
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}}}
]
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.batch_write_item(
table_name,
delete_items=items,
return_consumed_capacity='TOTAL',
return_item_collection_metrics='SIZE'
)
params = {
'return_consumed_capacity': 'TOTAL',
'return_item_collection_metrics': 'SIZE',
'request_items': {
'Thread': [
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-0'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-1'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-2'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-3'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-4'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-5'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-6'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-7'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-8'}}}},
{'DeleteRequest': {'Key': {'ForumName': {'S': 'FooForum'}, 'Subject': {'S': 'thread-9'}}}}
]
}
}
self.assertEqual(req.call_args[1], params)
def test_update_item(self):
"""
Connection.update_item
"""
conn = Connection()
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(self.test_table_name)
self.assertRaises(ValueError, conn.update_item, self.test_table_name, 'foo-key')
attr_updates = {
'Subject': {
'Value': 'foo-subject',
'Action': 'PUT'
},
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
UpdateError,
conn.update_item,
self.test_table_name,
'foo-key',
attribute_updates=attr_updates,
range_key='foo-range-key',
)
with patch(PATCH_METHOD) as req:
bad_attr_updates = {
'Subject': {
'Value': 'foo-subject',
'Action': 'BADACTION'
},
}
req.return_value = HttpOK(), {}
self.assertRaises(
ValueError,
conn.update_item,
self.test_table_name,
'foo-key',
attribute_updates=bad_attr_updates,
range_key='foo-range-key',
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.update_item(
self.test_table_name,
'foo-key',
return_consumed_capacity='TOTAL',
return_item_collection_metrics='NONE',
return_values='ALL_NEW',
expected={'Forum': {'Exists': False}},
attribute_updates=attr_updates,
range_key='foo-range-key',
)
params = {
'return_values': 'ALL_NEW',
'return_item_collection_metrics': 'NONE',
'return_consumed_capacity': 'TOTAL',
'key': {
'ForumName': {
'S': 'foo-key'
},
'Subject': {
'S': 'foo-range-key'
}
},
'expected': {
'Forum': {
'Exists': False
}
},
'attribute_updates': {
'Subject': {
'Value': {
'S': 'foo-subject'
},
'Action': 'PUT'
}
},
'table_name': 'ci-table'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.update_item(
self.test_table_name,
'foo-key',
attribute_updates=attr_updates,
range_key='foo-range-key',
)
params = {
'key': {
'ForumName': {
'S': 'foo-key'
},
'Subject': {
'S': 'foo-range-key'
}
},
'attribute_updates': {
'Subject': {
'Value': {
'S': 'foo-subject'
},
'Action': 'PUT'
}
},
'return_consumed_capacity': 'TOTAL',
'table_name': 'ci-table'
}
self.assertEqual(req.call_args[1], params)
attr_updates = {
'Subject': {
'Value': {'S': 'foo-subject'},
'Action': 'PUT'
},
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.update_item(
self.test_table_name,
'foo-key',
attribute_updates=attr_updates,
range_key='foo-range-key',
)
params = {
'key': {
'ForumName': {
'S': 'foo-key'
},
'Subject': {
'S': 'foo-range-key'
}
},
'attribute_updates': {
'Subject': {
'Value': {
'S': 'foo-subject'
},
'Action': 'PUT'
}
},
'return_consumed_capacity': 'TOTAL',
'table_name': 'ci-table'
}
self.assertEqual(req.call_args[1], params)
attr_updates = {
'Subject': {
'Value': {'N': '1'},
'Action': 'ADD'
},
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.update_item(
self.test_table_name,
'foo-key',
attribute_updates=attr_updates,
range_key='foo-range-key',
)
params = {
'key': {
'ForumName': {
'S': 'foo-key'
},
'Subject': {
'S': 'foo-range-key'
}
},
'attribute_updates': {
'Subject': {
'Value': {
'N': '1'
},
'Action': 'ADD'
}
},
'return_consumed_capacity': 'TOTAL',
'table_name': 'ci-table'
}
self.assertEqual(req.call_args[1], params)
def test_commit__not_implemented(self):
t = Transaction(connection=Connection())
with pytest.raises(NotImplementedError):
t._commit()
def test_commit_auto_version_condition(self, mocker):
connection = Connection()
mock_connection_transact_write = mocker.patch.object(
connection, 'transact_write_items')
self._handle_transact_write(connection, False)
expected_condition_checks = [{
'ConditionExpression': 'attribute_not_exists (#0)',
'ExpressionAttributeNames': {
'#0': 'mock_hash'
},
'Key': {
'MockHash': {
'N': '1'
},
'MockRange': {
'N': '3'
}
},
'TableName': 'mock'
}]
expected_deletes = [{
'Key': {
'MockHash': {
'N': '2'
},
'MockRange': {
'N': '4'
}
},
'TableName': 'mock'
}]
expected_puts = [{
'Item': {
'MockHash': {
'N': '3'
},
'MockRange': {
'N': '5'
},
'mock_version': {
'N': '1'
}
},
'TableName': 'mock'
}]
expected_updates = [{
'TableName': 'mock',
'Key': {
'MockHash': {
'N': '4'
},
'MockRange': {
'N': '6'
}
},
'ReturnValuesOnConditionCheckFailure': 'ALL_OLD',
'UpdateExpression': 'SET #0 = :0 ADD #1 :1',
'ExpressionAttributeNames': {
'#0': 'mock_toot',
'#1': 'mock_version'
},
'ExpressionAttributeValues': {
':0': {
'S': 'hello'
},
':1': {
'N': '1'
}
}
}]
mock_connection_transact_write.assert_called_once_with(
condition_check_items=expected_condition_checks,
delete_items=expected_deletes,
put_items=expected_puts,
update_items=expected_updates,
client_request_token=None,
return_consumed_capacity=None,
return_item_collection_metrics=None)
office_out_of_date.employees.remove(garrett)
with assert_condition_check_fails():
office_out_of_date.save()
# After refreshing the local copy the operation will succeed.
office_out_of_date.refresh()
office_out_of_date.employees.remove(garrett)
office_out_of_date.save()
assert office_out_of_date.version == 3
# Condition check fails for delete.
with assert_condition_check_fails():
office.delete()
# Example failed transactions.
connection = Connection(host='http://localhost:8000')
with assert_condition_check_fails(), TransactWrite(connection=connection) as transaction:
transaction.save(Office(office.office_id, name='newer name', employees=[]))
with assert_condition_check_fails(), TransactWrite(connection=connection) as transaction:
transaction.update(
Office(office.office_id, name='newer name', employees=[]),
actions=[
Office.name.set('Newer Office Name'),
]
)
with assert_condition_check_fails(), TransactWrite(connection=connection) as transaction:
transaction.delete(Office(office.office_id, name='newer name', employees=[]))
office_out_of_date.employees.remove(garrett)
with assert_condition_check_fails():
office_out_of_date.save()
# After refreshing the local copy the operation will succeed.
office_out_of_date.refresh()
office_out_of_date.employees.remove(garrett)
office_out_of_date.save()
assert office_out_of_date.version == 3
# Condition check fails for delete.
with assert_condition_check_fails():
office.delete()
# Example failed transactions.
connection = Connection(host='http://dynamodb:8000')
with assert_condition_check_fails(), TransactWrite(
connection=connection) as transaction:
transaction.save(Office(office.office_id, name='newer name', employees=[]))
with assert_condition_check_fails(), TransactWrite(
connection=connection) as transaction:
transaction.update(Office(office.office_id,
name='newer name',
employees=[]),
actions=[
Office.name.set('Newer Office Name'),
])
with assert_condition_check_fails(), TransactWrite(
import random
from django.utils import timezone
from django.conf import settings
from organisations.serializers import OrganisationSerializer, ProjectSerializer, EmployeeSerializer, \
ProjectEmployeeSerializer
from organisations.models import Organisation, Project, Employee
from faker import Faker
from pynamodb.connection import Connection
conn = Connection(host='http://localhost:8080', region='localhost')
fake = Faker()
def generate_organisations():
for _ in range(40):
print("Generating organisation")
serializer = OrganisationSerializer(
data=dict(name=fake.word(), tier=fake.word()))
if serializer.is_valid():
serializer.create(serializer.validated_data)
def update_organisation_tiers():
organisations = Organisation.scan()
tiers = ['tier_one', 'tier_two', 'tier_three']
def test_scan(self):
"""
Connection.scan
"""
conn = Connection()
table_name = 'Thread'
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.scan(
table_name,
segment=0,
total_segments=22,
)
params = {
'return_consumed_capacity': 'TOTAL',
'table_name': table_name,
'segment': 0,
'total_segments': 22,
}
self.assertDictEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.scan(
table_name,
return_consumed_capacity='TOTAL',
exclusive_start_key="FooForum",
limit=1,
segment=2,
total_segments=4,
attributes_to_get=['ForumName']
)
params = {
'attributes_to_get': ['ForumName'],
'exclusive_start_key': {
"ForumName": {
"S": "FooForum"
}
},
'table_name': table_name,
'limit': 1,
'segment': 2,
'total_segments': 4,
'return_consumed_capacity': 'TOTAL'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.scan(
table_name,
)
params = {
'return_consumed_capacity': 'TOTAL',
'table_name': table_name
}
self.assertEqual(req.call_args[1], params)
kwargs = {
'scan_filter': {
'ForumName': {
'ComparisonOperator': 'BadOperator',
'AttributeValueList': ['Foo']
}
}
}
self.assertRaises(
ValueError,
conn.scan,
table_name,
**kwargs)
kwargs = {
'scan_filter': {
'ForumName': {
'ComparisonOperator': 'BEGINS_WITH',
'AttributeValueList': ['Foo']
}
}
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
ScanError,
conn.scan,
table_name,
**kwargs)
kwargs = {
'scan_filter': {
'ForumName': {
'ComparisonOperator': 'BEGINS_WITH',
'AttributeValueList': ['Foo']
}
}
}
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.scan(
table_name,
**kwargs
)
params = {
'return_consumed_capacity': 'TOTAL',
'table_name': table_name,
'scan_filter': {
'ForumName': {
'AttributeValueList': [
{'S': 'Foo'}
],
'ComparisonOperator': 'BEGINS_WITH'
}
}
}
self.assertEqual(req.call_args[1], params)
def test_commit(self, mocker):
connection = Connection()
mock_connection_transact_write = mocker.patch.object(
connection, 'transact_write_items')
with patch(PATCH_METHOD) as req:
req.return_value = MOCK_TABLE_DESCRIPTOR
with TransactWrite(connection=connection) as t:
t.condition_check(
MockModel,
1,
3,
condition=(MockModel.mock_hash.does_not_exist()))
t.delete(MockModel(2, 4))
t.save(MockModel(3, 5))
t.update(MockModel(4, 6),
actions=[MockModel.mock_toot.set('hello')],
return_values='ALL_OLD')
expected_condition_checks = [{
'ConditionExpression': 'attribute_not_exists (#0)',
'ExpressionAttributeNames': {
'#0': 'mock_hash'
},
'Key': {
'MockHash': {
'N': '1'
},
'MockRange': {
'N': '3'
}
},
'TableName': 'mock'
}]
expected_deletes = [{
'ConditionExpression': 'attribute_not_exists (#0)',
'ExpressionAttributeNames': {
'#0': 'mock_version'
},
'Key': {
'MockHash': {
'N': '2'
},
'MockRange': {
'N': '4'
}
},
'TableName': 'mock'
}]
expected_puts = [{
'ConditionExpression': 'attribute_not_exists (#0)',
'ExpressionAttributeNames': {
'#0': 'mock_version'
},
'Item': {
'MockHash': {
'N': '3'
},
'MockRange': {
'N': '5'
},
'mock_version': {
'N': '1'
}
},
'TableName': 'mock'
}]
expected_updates = [{
'ConditionExpression': 'attribute_not_exists (#0)',
'TableName': 'mock',
'Key': {
'MockHash': {
'N': '4'
},
'MockRange': {
'N': '6'
}
},
'ReturnValuesOnConditionCheckFailure': 'ALL_OLD',
'UpdateExpression': 'SET #1 = :0, #0 = :1',
'ExpressionAttributeNames': {
'#0': 'mock_version',
'#1': 'mock_toot'
},
'ExpressionAttributeValues': {
':0': {
'S': 'hello'
},
':1': {
'N': '1'
}
}
}]
mock_connection_transact_write.assert_called_once_with(
condition_check_items=expected_condition_checks,
delete_items=expected_deletes,
put_items=expected_puts,
update_items=expected_updates,
client_request_token=None,
return_consumed_capacity=None,
return_item_collection_metrics=None)
def test_connection_integration(ddb_url):
table_name = 'pynamodb-ci-connection'
# For use with a fake dynamodb connection
# See: http://aws.amazon.com/dynamodb/developer-resources/
conn = Connection(host=ddb_url)
print(conn)
print("conn.describe_table...")
table = None
try:
table = conn.describe_table(table_name)
except TableDoesNotExist:
params = {
'read_capacity_units':
1,
'write_capacity_units':
1,
'attribute_definitions': [{
'attribute_type': STRING,
'attribute_name': 'Forum'
}, {
'attribute_type': STRING,
'attribute_name': 'Thread'
}, {
'attribute_type': STRING,
'attribute_name': 'AltKey'
}, {
'attribute_type': NUMBER,
'attribute_name': 'number'
}],
'key_schema': [{
'key_type': HASH,
'attribute_name': 'Forum'
}, {
'key_type': RANGE,
'attribute_name': 'Thread'
}],
'global_secondary_indexes': [{
'index_name':
'alt-index',
'key_schema': [{
'KeyType': 'HASH',
'AttributeName': 'AltKey'
}],
'projection': {
'ProjectionType': 'KEYS_ONLY'
},
'provisioned_throughput': {
'ReadCapacityUnits': 1,
'WriteCapacityUnits': 1,
}
}],
'local_secondary_indexes': [{
'index_name':
'view-index',
'key_schema': [{
'KeyType': 'HASH',
'AttributeName': 'Forum'
}, {
'KeyType': 'RANGE',
'AttributeName': 'AltKey'
}],
'projection': {
'ProjectionType': 'KEYS_ONLY'
}
}]
}
print("conn.create_table...")
conn.create_table(table_name, **params)
while table is None:
time.sleep(1)
table = conn.describe_table(table_name)
while table['TableStatus'] == 'CREATING':
time.sleep(2)
table = conn.describe_table(table_name)
print("conn.list_tables")
conn.list_tables()
print("conn.update_table...")
conn.update_table(table_name,
read_capacity_units=table.get(
PROVISIONED_THROUGHPUT).get(READ_CAPACITY_UNITS) + 1,
write_capacity_units=2)
table = conn.describe_table(table_name)
while table['TableStatus'] != 'ACTIVE':
time.sleep(2)
table = conn.describe_table(table_name)
print("conn.put_item")
conn.put_item(
table_name,
'item1-hash',
range_key='item1-range',
attributes={'foo': {
'S': 'bar'
}},
condition=NotExists(Path('Forum')),
)
conn.get_item(table_name, 'item1-hash', range_key='item1-range')
conn.delete_item(table_name, 'item1-hash', range_key='item1-range')
items = []
for i in range(10):
items.append({"Forum": "FooForum", "Thread": f"thread-{i}"})
print("conn.batch_write_items...")
conn.batch_write_item(table_name, put_items=items)
print("conn.batch_get_items...")
data = conn.batch_get_item(table_name, items)
print("conn.query...")
conn.query(
table_name,
"FooForum",
range_key_condition=(BeginsWith(Path('Thread'), Value('thread'))),
)
print("conn.scan...")
conn.scan(table_name, )
print("conn.delete_table...")
conn.delete_table(table_name)
"""
Runs tests against dynamodb
"""
from __future__ import print_function
import time
import config as cfg
from pynamodb.connection import Connection
from pynamodb.constants import PROVISIONED_THROUGHPUT, READ_CAPACITY_UNITS
from pynamodb.types import STRING, HASH, RANGE, NUMBER
table_name = 'pynamodb-ci'
# For use with a fake dynamodb connection
# See: http://aws.amazon.com/dynamodb/developer-resources/
conn = Connection(host=cfg.DYNAMODB_HOST)
print(conn)
print("conn.describe_table...")
table = conn.describe_table(table_name)
if table is None:
params = {
'read_capacity_units': 1,
'write_capacity_units': 1,
'attribute_definitions': [
{
'attribute_type': STRING,
'attribute_name': 'Forum'
},
{
'attribute_type': STRING,
'attribute_name': 'Thread'
def connection(ddb_url):
yield Connection(host=ddb_url)
def test_delete_item(self):
"""
Connection.delete_item
"""
conn = Connection(self.region)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(self.test_table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(DeleteError, conn.delete_item, self.test_table_name, "foo", "bar")
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.delete_item(
self.test_table_name,
"Amazon DynamoDB",
"How do I update multiple items?")
params = {
'return_consumed_capacity': 'TOTAL',
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'table_name': self.test_table_name}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.delete_item(
self.test_table_name,
"Amazon DynamoDB",
"How do I update multiple items?",
return_values='ALL_NEW'
)
params = {
'return_consumed_capacity': 'TOTAL',
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'table_name': self.test_table_name,
'return_values': 'ALL_NEW'
}
self.assertEqual(req.call_args[1], params)
self.assertRaises(
ValueError,
conn.delete_item,
self.test_table_name,
"foo",
"bar",
return_values='bad_values')
self.assertRaises(
ValueError,
conn.delete_item,
self.test_table_name,
"foo",
"bar",
return_consumed_capacity='badvalue')
self.assertRaises(
ValueError,
conn.delete_item,
self.test_table_name,
"foo",
"bar",
return_item_collection_metrics='badvalue')
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.delete_item(
self.test_table_name,
"Amazon DynamoDB",
"How do I update multiple items?",
return_consumed_capacity='TOTAL'
)
params = {
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'table_name': self.test_table_name,
'return_consumed_capacity': 'TOTAL'
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.delete_item(
self.test_table_name,
"Amazon DynamoDB",
"How do I update multiple items?",
return_item_collection_metrics='SIZE'
)
params = {
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'table_name': self.test_table_name,
'return_item_collection_metrics': 'SIZE',
'return_consumed_capacity': 'TOTAL'
}
self.assertEqual(req.call_args[1], params)
self.assertRaises(
ValueError,
conn.delete_item,
self.test_table_name,
"Foo", "Bar",
expected={'Bad': {'Value': False}}
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.delete_item(
self.test_table_name,
"Amazon DynamoDB",
"How do I update multiple items?",
expected={'ForumName': {'Exists': False}},
return_item_collection_metrics='SIZE'
)
params = {
'key': {
'ForumName': {
'S': 'Amazon DynamoDB'
},
'Subject': {
'S': 'How do I update multiple items?'
}
},
'expected': {
'ForumName': {
'Exists': False
}
},
'table_name': self.test_table_name,
'return_consumed_capacity': 'TOTAL',
'return_item_collection_metrics': 'SIZE'
}
self.assertEqual(req.call_args[1], params)
def dump_table(host, region, table_name, total_segments=1, hash_keys=None,
compress=False, parallelism=1, capacity_consumption=0.5):
if hash_keys is None:
hash_keys = []
capacity_consumption = max(0.01, capacity_consumption)
capacity_consumption = min(1.0, capacity_consumption)
connection = Connection(host=host, region=region)
desc = connection.describe_table(table_name)
if desc is None:
sys.stderr.writelines(["Table does not exist."])
sys.exit(-1)
total_items = desc['ItemCount']
total_capacity = desc[PROVISIONED_THROUGHPUT][READ_CAPACITY_UNITS]
capacity_per_process = max(
1.0,
(capacity_consumption * total_capacity) / float(parallelism)
)
queue = multiprocessing.Queue()
pool = multiprocessing.Pool(
processes=parallelism,
initializer=pool_init,
initargs=[queue, capacity_per_process, host, region, compress, table_name]
)
# Keys or segments?
if len(hash_keys):
for key in hash_keys:
pool.apply_async(dump_part, [key])
num_to_do = len(hash_keys)
else:
for segment in range(total_segments):
pool.apply_async(dump_part, [segment, total_segments])
num_to_do = total_segments
num_complete = 0
items_dumped = 0
while True:
sleep(1)
while not queue.empty():
update = queue.get()
if update == 'complete':
num_complete += 1
else:
items_dumped += update
print("{}/~{} items dumped - {}/{} {}.".format(
items_dumped,
total_items,
num_complete,
num_to_do,
"keys" if len(hash_keys) else "segments"
))
if num_complete == num_to_do:
break
pool.close()
pool.join()
print("Done.")
def test_condition_check__no_condition(self):
with pytest.raises(TypeError):
with TransactWrite(connection=Connection()) as transaction:
transaction.condition_check(MockModel,
hash_key=1,
condition=None)
def updateOrder(request):
body = request.body.decode('utf-8')
params = json.loads(body)
shop_domain = params.get('shop_domain', 'nothing')
order_status = params.get('order_status', 'nothing') #use in future
is_digital = params.get('is_digital', False) #use in future
shipping_company = params.get('shipping_company',
'nothing') #use in future
tracking_no = params.get('tracking_no', 'nothing') #use in future
print(shop_domain)
conn = Connection(region='us-east-1')
table = conn.list_tables()
personal_data = conn.query('updateOrder', shop_domain)
if personal_data == {}:
data = {'status': 'Failure'}
dump = json.dumps(data)
return HttpResponse(dump, content_type='application/json')
print(personal_data)
for item in personal_data['Items']:
print(item['username']['S'])
comment_numper = 4
driver = webdriver.Chrome('chromedriver')
driver.get("https://news.ycombinator.com/news")
driver.maximize_window()
time.sleep(1)
elem = driver.find_element_by_xpath(
'//a[@href="login?goto=news"]'
) # driver.find_element_by_link_text("login")
ActionChains(driver).move_to_element(elem).click().perform()
elem = driver.find_element_by_name("acct")
ActionChains(driver).move_to_element(elem).click().perform()
ActionChains(driver).send_keys(item['username']['S']).perform()
elem = driver.find_element_by_name("pw")
ActionChains(driver).move_to_element(elem).click().perform()
ActionChains(driver).send_keys(item['password']['S']).perform()
ActionChains(driver).send_keys(Keys.RETURN).perform()
driver.get("https://news.ycombinator.com/news")
time.sleep(1)
upvote_elems = driver.find_elements_by_class_name("votearrow")
ActionChains(driver).move_to_element(upvote_elems[random.randint(
0,
len(upvote_elems) - 1)]).click().perform()
time.sleep(1)
comment_elems = driver.find_elements_by_partial_link_text("comment")
ActionChains(driver).move_to_element(
comment_elems[int(comment_numper)]).click().perform()
write_comment_element = driver.find_element_by_xpath(
'//textarea[@name="text"]')
ActionChains(driver).move_to_element(
write_comment_element).click().perform()
ActionChains(driver).send_keys(
'This is one of the best test comments ever. Its bigly.').perform(
)
submit_element = driver.find_element_by_xpath(
'//input[@type="submit"]')
ActionChains(driver).move_to_element(submit_element).click().perform()
time.sleep(1)
driver.get("https://news.ycombinator.com/news")
logout_elem = driver.find_element_by_id("logout")
ActionChains(driver).move_to_element(logout_elem).click().perform()
driver.close()
data = {'status': 'Success'}
dump = json.dumps(data)
return HttpResponse(dump, content_type='application/json')
def test_put_item(self):
"""
Connection.put_item
"""
conn = Connection(self.region)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), DESCRIBE_TABLE_DATA
conn.describe_table(self.test_table_name)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.put_item(
self.test_table_name,
'foo-key',
range_key='foo-range-key',
return_consumed_capacity='TOTAL',
return_item_collection_metrics='SIZE',
return_values='ALL_NEW',
attributes={'ForumName': 'foo-value'}
)
params = {
'return_values': 'ALL_NEW',
'return_consumed_capacity': 'TOTAL',
'return_item_collection_metrics': 'SIZE',
'table_name': self.test_table_name,
'item': {
'ForumName': {
'S': 'foo-value'
},
'Subject': {
'S': 'foo-range-key'
}
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpBadRequest(), {}
self.assertRaises(
PutError,
conn.put_item,
self.test_table_name,
'foo-key',
range_key='foo-range-key',
attributes={'ForumName': 'foo-value'}
)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.put_item(
self.test_table_name,
'foo-key',
range_key='foo-range-key',
attributes={'ForumName': 'foo-value'}
)
params = {'table_name': self.test_table_name,
'return_consumed_capacity': 'TOTAL',
'item': {'ForumName': {'S': 'foo-value'}, 'Subject': {'S': 'foo-range-key'}}}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.put_item(
self.test_table_name,
'foo-key',
range_key='foo-range-key',
attributes={'ForumName': 'foo-value'}
)
params = {
'return_consumed_capacity': 'TOTAL',
'item': {
'ForumName': {
'S': 'foo-value'
},
'Subject': {
'S': 'foo-range-key'
}
},
'table_name': self.test_table_name
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.put_item(
self.test_table_name,
'item1-hash',
range_key='item1-range',
attributes={'foo': {'S': 'bar'}},
expected={'Forum': {'Exists': False}}
)
params = {
'return_consumed_capacity': 'TOTAL',
'table_name': self.test_table_name,
'expected': {
'Forum': {
'Exists': False
}
},
'item': {
'ForumName': {
'S': 'item1-hash'
},
'foo': {
'S': 'bar'
},
'Subject': {
'S': 'item1-range'
}
}
}
self.assertEqual(req.call_args[1], params)
with patch(PATCH_METHOD) as req:
req.return_value = HttpOK(), {}
conn.put_item(
self.test_table_name,
'item1-hash',
range_key='item1-range',
attributes={'foo': {'S': 'bar'}},
expected={'ForumName': {'Value': 'item1-hash'}}
)
params = {
'table_name': self.test_table_name,
'expected': {
'ForumName': {
'Value': {
'S': 'item1-hash'
}
}
},
'return_consumed_capacity': 'TOTAL',
'item': {
'ForumName': {
'S': 'item1-hash'
},
'foo': {
'S': 'bar'
},
'Subject': {
'S': 'item1-range'
}
}
}
self.assertEqual(req.call_args[1], params)
# https://pynamodb.readthedocs.io/en/latest/low_level.html
# Creating a connection
# Creating a connection is simple
from pynamodb.connection import Connection
conn = Connection()
# You can specify a different DynamoDB url
conn = Connection(host='http://alternative-domain/')
# By default, PynamoDB will connect to the us-east-1 region, but you can specify a different one.
conn = Connection(region='us-west-1')
# Modifying tables
# You can easily list tables:
conn.list_tables()
# or delete a table:
# conn.delete_table('Thread')
# If you want to change the capacity of a table, that can be done as well:
conn.update_table('Thread', read_capacity_units=20, write_capacity_units=20)
# You can create tables as well, although the syntax is verbose. You should really use the model API instead,
# but here is a low level example to demonstrate the point:
kwargs = {
def list_table():
from pynamodb.connection import Connection
conn = Connection(host=DB_HOST, region=REGION)
tables = conn.list_tables()
print(tables)
return tables
"""
Examples using a connection
"""
from __future__ import print_function
from pynamodb.connection import Connection
# Get a connection
conn = Connection(host='http://localhost:8000')
print(conn)
# List tables
print(conn.list_tables())
# Describe a table
print(conn.describe_table('Thread'))
# Get an item
print(conn.get_item('Thread', 'hash-key', 'range-key'))
# Put an item
conn.put_item('Thread', 'hash-key', 'range-key', attributes={'forum_name': 'value', 'subject': 'value'})
# Delete an item
conn.delete_item('Thread', 'hash-key', 'range-key')
"""
Examples using a connection
"""
from pynamodb.connection import Connection
# Get a connection
conn = Connection(host='http://localhost:8000')
print(conn)
# List tables
print(conn.list_tables())
# Describe a table
print(conn.describe_table('Thread'))
# Get an item
print(conn.get_item('Thread', 'hash-key', 'range-key'))
# Put an item
conn.put_item('Thread',
'hash-key',
'range-key',
attributes={
'forum_name': 'value',
'subject': 'value'
})
# Delete an item
conn.delete_item('Thread', 'hash-key', 'range-key')
"""
Runs tests against dynamodb
"""
from __future__ import print_function
import time
import config as cfg
from pynamodb.connection import Connection
from pynamodb.constants import PROVISIONED_THROUGHPUT, READ_CAPACITY_UNITS
from pynamodb.types import STRING, HASH, RANGE, NUMBER
table_name = 'pynamodb-ci'
# For use with a fake dynamodb connection
# See: http://aws.amazon.com/dynamodb/developer-resources/
conn = Connection(host=cfg.DYNAMODB_HOST)
print(conn)
print("conn.describe_table...")
table = conn.describe_table(table_name)
if table is None:
params = {
'read_capacity_units':
1,
'write_capacity_units':
1,
'attribute_definitions': [{
'attribute_type': STRING,
'attribute_name': 'Forum'
}, {
'attribute_type': STRING,
'attribute_name': 'Thread'
