def test_create_table_throughput(self):
"""Create a table and set throughput"""
hash_key = DynamoKey("id", data_type=STRING)
throughput = Throughput(8, 2)
table = Table("foobar", hash_key, throughput=throughput)
self.dynamo.create_table("foobar", hash_key=hash_key, throughput=throughput)
desc = self.dynamo.describe_table("foobar")
self.assertEqual(desc, table)
def test_update_table_throughput(self):
""" Update the table throughput """
hash_key = DynamoKey('id', data_type=STRING)
self.dynamo.create_table('foobar', hash_key=hash_key)
tp = Throughput(2, 1)
self.dynamo.update_table('foobar', throughput=tp)
table = self.dynamo.describe_table('foobar')
self.assertEqual(table.throughput, tp)
def test_capacity(self):
"""Can return consumed capacity"""
conn = MagicMock()
response_cap = {
"TableName": "foobar",
"ReadCapacityUnits": 6,
"WriteCapacityUnits": 7,
"Table": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 2,
},
"LocalSecondaryIndexes": {
"l-index": {
"ReadCapacityUnits": 2,
"WriteCapacityUnits": 3,
},
},
"GlobalSecondaryIndexes": {
"g-index": {
"ReadCapacityUnits": 3,
"WriteCapacityUnits": 4,
},
},
}
response = {
"Responses": [],
"ConsumedCapacity": [response_cap],
}
capacity = ConsumedCapacity.from_response(response_cap)
response["consumed_capacity"] = [capacity]
with patch.object(self.dynamo, "client") as client:
client.transact_get_items.return_value = response
ret = self.dynamo.txn_get("foobar", [{"id": "a"}])
list(ret)
assert ret.consumed_capacity is not None
cap = ret.consumed_capacity["foobar"]
assert cap is not None
assert cap.table_capacity is not None
assert cap.local_index_capacity is not None
assert cap.global_index_capacity is not None
self.assertEqual(cap.total, Throughput(6, 7))
self.assertEqual(cap.table_capacity, Throughput(1, 2))
self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
self.assertEqual(cap.global_index_capacity["g-index"],
Throughput(3, 4))
def test_update_table_throughput(self):
"""Update the table throughput"""
hash_key = DynamoKey("id", data_type=STRING)
self.dynamo.create_table("foobar", hash_key=hash_key, throughput=(1, 1))
tp = Throughput(3, 4)
self.dynamo.update_table("foobar", throughput=tp)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.throughput, tp)
def test_capacity(self):
"""Can return consumed capacity"""
ret = {
"Responses": {
"foo": [],
},
"ConsumedCapacity": [
{
"TableName": "foobar",
"ReadCapacityUnits": 6,
"WriteCapacityUnits": 7,
"Table": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 2,
},
"LocalSecondaryIndexes": {
"l-index": {
"ReadCapacityUnits": 2,
"WriteCapacityUnits": 3,
},
},
"GlobalSecondaryIndexes": {
"g-index": {
"ReadCapacityUnits": 3,
"WriteCapacityUnits": 4,
},
},
}
],
}
with patch.object(self.dynamo.client, "batch_write_item", return_value=ret):
batch = self.dynamo.batch_write("foobar", return_capacity="INDEXES")
with batch:
batch.put({"id": "a"})
cap = batch.consumed_capacity
assert cap is not None
assert cap.table_capacity is not None
assert cap.local_index_capacity is not None
assert cap.global_index_capacity is not None
self.assertEqual(cap.total, Throughput(6, 7))
self.assertEqual(cap.table_capacity, Throughput(1, 2))
self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
self.assertEqual(cap.global_index_capacity["g-index"], Throughput(3, 4))
def test_alter_throughput_from_defaults(self):
""" Updates index throughput values by comparing the current values to those from the actual database.
This might not be a great idea. create_schema doesn't seem to perform any updates with read/write
throughput capacity changes. Not really sure if this should.
"""
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(len(table.global_indexes), 1)
self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))
# Simulating adding an index later on.
WidgetToAddIndex.meta_.global_indexes[
0] = WidgetToAddIndex.meta_.global_indexes[0].throughput(read=10,
write=11)
WidgetToAddIndex.meta_.post_create()
WidgetToAddIndex.meta_.validate_model()
WidgetToAddIndex.meta_.post_validate()
changed = self.engine.update_schema()
self.assertListEqual(
changed,
[WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(table.global_indexes[0].throughput,
Throughput(10, 11))
self.assertEqual(len(table.global_indexes), 1)
one = WidgetToAddIndex("one-1", "one-2", 1)
two = WidgetToAddIndex("one-2", "test", 2)
self.engine.save(one)
self.engine.save(two)
result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
WidgetToAddIndex.string2 == 'test').all()
self.assertEqual(len(result), 1)
self.assertNotEqual(result[0], one)
self.assertEqual(result[0], two)
def test_update_global_index_throughput_old(self):
""" Update throughput on a global index OLD API """
hash_key = DynamoKey('id', data_type=STRING)
index_field = DynamoKey('name')
index = GlobalIndex.all('name-index', index_field)
self.dynamo.create_table('foobar',
hash_key=hash_key,
global_indexes=[index])
tp = Throughput(2, 1)
self.dynamo.update_table('foobar', global_indexes={'name-index': tp})
table = self.dynamo.describe_table('foobar')
self.assertEqual(table.global_indexes[0].throughput, tp)
def test_update_multiple_throughputs(self):
"""Update table and global index throughputs"""
hash_key = DynamoKey("id", data_type=STRING)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field, throughput=(2, 3))
self.dynamo.create_table(
"foobar",
hash_key=hash_key,
global_indexes=[index],
throughput=Throughput(1, 1),
)
tp = Throughput(3, 4)
self.dynamo.update_table(
"foobar",
throughput=tp,
index_updates=[IndexUpdate.update("name-index", tp)],
)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.throughput, tp)
self.assertEqual(table.global_indexes[0].throughput, tp)
def test_create_global_index_throughput(self):
"""Create a table and set throughput on global index"""
hash_key = DynamoKey("id", data_type=STRING)
throughput = Throughput(8, 2)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field, throughput=throughput)
table = Table("foobar", hash_key, global_indexes=[index], throughput=throughput)
self.dynamo.create_table(
"foobar", hash_key=hash_key, global_indexes=[index], throughput=throughput
)
desc = self.dynamo.describe_table("foobar")
self.assertEqual(desc, table)
def test_alter_throughput_directly(self):
""" Tests that throughput provisioning specified directly at update time are applied to the index.
"""
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(len(table.global_indexes), 1)
self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))
changed = self.engine.update_schema(
throughput={
WidgetToAddIndex.meta_.ddb_tablename(): {
"gindex": {
"read": 12,
"write": 13
}
}
})
self.assertListEqual(
changed,
[WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(table.global_indexes[0].throughput,
Throughput(12, 13))
self.assertEqual(len(table.global_indexes), 1)
one = WidgetToAddIndex("one-1", "one-2", 1)
two = WidgetToAddIndex("one-2", "test", 2)
self.engine.save(one)
self.engine.save(two)
result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
WidgetToAddIndex.string2 == 'test').all()
self.assertEqual(len(result), 1)
self.assertNotEqual(result[0], one)
self.assertEqual(result[0], two)
def test_update_index_throughput(self):
"""Update the throughput on a global index"""
hash_key = DynamoKey("id", data_type=STRING)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field)
self.dynamo.create_table("foobar", hash_key=hash_key, global_indexes=[index])
tp = Throughput(2, 1)
self.dynamo.update_table(
"foobar", index_updates=[IndexUpdate.update("name-index", tp)]
)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.global_indexes[0].throughput, tp)
def __init__(self, model):
self.model = model
self._name = model.__name__
self.global_indexes = []
self.orderings = []
self.throughput = Throughput()
self._abstract = False
self.__dict__.update(model.__metadata__)
# Allow throughput to be specified as read/write in a dict
# pylint: disable=E1134
if isinstance(self.throughput, dict):
self.throughput = Throughput(**self.throughput)
# pylint: enable=E1134
self.name = self._name
self.fields = {}
self.hash_key = None
self.range_key = None
self.related_fields = defaultdict(set)
self.all_global_indexes = set()
for gindex in self.global_indexes:
self.all_global_indexes.add(gindex.hash_key)
if gindex.range_key is not None:
self.all_global_indexes.add(gindex.range_key)
def test_update_billing_mode(self):
"""Update a table billing mode"""
hash_key = DynamoKey("id", data_type=STRING)
table = self.dynamo.create_table(
"foobar", hash_key=hash_key, billing_mode=PAY_PER_REQUEST
)
assert table is not None
self.assertEqual(table.billing_mode, PAY_PER_REQUEST)
new_table = self.dynamo.update_table(
"foobar", billing_mode=PROVISIONED, throughput=(2, 3)
)
assert new_table is not None
self.assertEqual(new_table.billing_mode, PROVISIONED)
self.assertEqual(new_table.throughput, Throughput(2, 3))
def update_dynamo_schema(self,
connection,
test=False,
wait=False,
throughput=None,
namespace=()):
"""
Updates all Dynamo table global indexes for this model
Parameters
----------
connection : :class:`~dynamo3.DynamoDBConnection`
test : bool, optional
If True, don't actually create the table (default False)
wait : bool, optional
If True, block until table has been created (default False)
throughput : dict, optional
The throughput of the table and global indexes. Has the keys 'read'
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
namespace : str or tuple, optional
The namespace of the table
Returns
-------
table : str
Table name that altered, or None if nothing altered
"""
if self.abstract:
return None
tablename = self.ddb_tablename(namespace)
global_indexes = []
for gindex in self.global_indexes:
index = gindex.get_ddb_index(self.fields)
if throughput is not None and gindex.name in throughput:
index.throughput = Throughput(**throughput[gindex.name])
global_indexes.append(index)
if not global_indexes:
return None
table = connection.describe_table(tablename)
if not table:
return None
expected_indexes = {}
for i in global_indexes:
expected_indexes[i.name] = i
actual_indexes = {}
for i in table.global_indexes:
actual_indexes[i.name] = i
missing_index_names = set(expected_indexes.keys()) - set(
actual_indexes.keys())
missing_indexes = [expected_indexes[i] for i in missing_index_names]
updates = [IndexUpdate.create(index) for index in missing_indexes]
update_indexes_name = set(expected_indexes.keys()) & set(
actual_indexes.keys())
update_indexes = [
expected_indexes[i] for i in update_indexes_name
if actual_indexes[i].throughput != expected_indexes[i].throughput
]
updates.extend([
IndexUpdate.update(index.name, index.throughput)
for index in update_indexes
])
if not updates:
return None
if not test:
connection.update_table(tablename, index_updates=updates)
if wait:
desc = connection.describe_table(tablename)
while desc.status != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(tablename)
return tablename
def create_dynamo_schema(self,
connection,
tablenames=None,
test=False,
wait=False,
throughput=None,
namespace=()):
"""
Create all Dynamo tables for this model
Parameters
----------
connection : :class:`~dynamo3.DynamoDBConnection`
tablenames : list, optional
List of tables that already exist. Will call 'describe' if not
provided.
test : bool, optional
If True, don't actually create the table (default False)
wait : bool, optional
If True, block until table has been created (default False)
throughput : dict, optional
The throughput of the table and global indexes. Has the keys 'read'
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
namespace : str or tuple, optional
The namespace of the table
Returns
-------
table : str
Table name that was created, or None if nothing created
"""
if self.abstract:
return None
if tablenames is None:
tablenames = set(connection.list_tables())
tablename = self.ddb_tablename(namespace)
if tablename in tablenames:
return None
elif test:
return tablename
indexes = []
global_indexes = []
hash_key = None
if throughput is not None:
table_throughput = Throughput(throughput['read'],
throughput['write'])
else:
table_throughput = self.throughput
hash_key = DynamoKey(self.hash_key.name,
data_type=self.hash_key.ddb_data_type)
range_key = None
if self.range_key is not None:
range_key = DynamoKey(self.range_key.name,
data_type=self.range_key.ddb_data_type)
for field in six.itervalues(self.fields):
if field.index:
idx = field.get_ddb_index()
indexes.append(idx)
for gindex in self.global_indexes:
index = gindex.get_ddb_index(self.fields)
if throughput is not None and gindex.name in throughput:
index.throughput = Throughput(**throughput[gindex.name])
global_indexes.append(index)
if not test:
connection.create_table(tablename, hash_key, range_key, indexes,
global_indexes, table_throughput)
if wait:
desc = connection.describe_table(tablename)
while desc.status != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(tablename)
return tablename