def _update_throughput(self, tablename, read, write, index):
""" Update the throughput on a table or index """
def get_desc():
""" Get the table or global index description """
desc = self.describe(tablename, refresh=True, require=True)
if index is not None:
return desc.global_indexes[index]
return desc
desc = get_desc()
def num_or_star(value):
""" Convert * to 0, otherwise resolve a number """
return 0 if value == "*" else resolve(value)
read = num_or_star(read)
write = num_or_star(write)
if read <= 0:
read = desc.read_throughput
if write <= 0:
write = desc.write_throughput
throughput = Throughput(read, write)
kwargs = {}
if index:
kwargs["global_indexes"] = {index: throughput}
else:
kwargs["throughput"] = throughput
self.connection.update_table(tablename, **kwargs)
desc = get_desc()
while desc.status == "UPDATING": # pragma: no cover
time.sleep(5)
desc = get_desc()
def throughput(self, read=5, write=5):
"""
Set the index throughput
Parameters
----------
read : int, optional
Amount of read throughput (default 5)
write : int, optional
Amount of write throughput (default 5)
Notes
-----
This is meant to be used as a chain::
class MyModel(Model):
__metadata__ = {
'global_indexes': [
GlobalIndex('myindex', 'hkey', 'rkey').throughput(5, 2)
]
}
"""
self._throughput = Throughput(read, write)
return self
def _create(self, tree):
""" Run a SELECT statement """
tablename = tree.table
indexes = []
global_indexes = []
hash_key = None
range_key = None
attrs = {}
for declaration in tree.attrs:
name, type_ = declaration[:2]
if len(declaration) > 2:
index = declaration[2]
else:
index = None
if index is not None:
if index[0] == "HASH":
field = hash_key = DynamoKey(name, data_type=TYPES[type_])
elif index[0] == "RANGE":
field = range_key = DynamoKey(name, data_type=TYPES[type_])
else:
index_type = index[0]
kwargs = {}
if index_type[0] in ("ALL", "INDEX"):
factory = LocalIndex.all
elif index_type[0] == "KEYS":
factory = LocalIndex.keys
elif index_type[0] == "INCLUDE":
factory = LocalIndex.include
kwargs["includes"] = [
resolve(v) for v in index.include_vars
]
index_name = resolve(index[1])
field = DynamoKey(name, data_type=TYPES[type_])
idx = factory(index_name, field, **kwargs)
indexes.append(idx)
else:
field = DynamoKey(name, data_type=TYPES[type_])
attrs[field.name] = field
for gindex in tree.global_indexes:
global_indexes.append(self._parse_global_index(gindex, attrs))
throughput = None
if tree.throughput:
throughput = Throughput(*map(resolve, tree.throughput))
try:
ret = self.connection.create_table(
tablename,
hash_key,
range_key,
indexes=indexes,
global_indexes=global_indexes,
throughput=throughput,
)
except DynamoDBError as e:
if e.kwargs["Code"] == "ResourceInUseException" or tree.not_exists:
return False
raise
return True
def __init__(self, name, hash_key, range_key=None):
self.name = name
self.hash_key = hash_key
self.range_key = range_key
self._throughput = Throughput()
self.ddb_index = GIndex.all
self.kwargs = {}
def test_create_table_throughput(self):
"""Create a table and set throughput"""
hash_key = DynamoKey("id", data_type=STRING)
throughput = Throughput(8, 2)
table = Table("foobar", hash_key, throughput=throughput)
self.dynamo.create_table("foobar", hash_key=hash_key, throughput=throughput)
desc = self.dynamo.describe_table("foobar")
self.assertEqual(desc, table)
def test_update_table_throughput(self):
""" Update the table throughput """
hash_key = DynamoKey('id', data_type=STRING)
self.dynamo.create_table('foobar', hash_key=hash_key)
tp = Throughput(2, 1)
self.dynamo.update_table('foobar', throughput=tp)
table = self.dynamo.describe_table('foobar')
self.assertEqual(table.throughput, tp)
def test_capacity(self):
"""Can return consumed capacity"""
conn = MagicMock()
response_cap = {
"TableName": "foobar",
"ReadCapacityUnits": 6,
"WriteCapacityUnits": 7,
"Table": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 2,
},
"LocalSecondaryIndexes": {
"l-index": {
"ReadCapacityUnits": 2,
"WriteCapacityUnits": 3,
},
},
"GlobalSecondaryIndexes": {
"g-index": {
"ReadCapacityUnits": 3,
"WriteCapacityUnits": 4,
},
},
}
response = {
"Responses": [],
"ConsumedCapacity": [response_cap],
}
capacity = ConsumedCapacity.from_response(response_cap)
response["consumed_capacity"] = [capacity]
with patch.object(self.dynamo, "client") as client:
client.transact_get_items.return_value = response
ret = self.dynamo.txn_get("foobar", [{"id": "a"}])
list(ret)
assert ret.consumed_capacity is not None
cap = ret.consumed_capacity["foobar"]
assert cap is not None
assert cap.table_capacity is not None
assert cap.local_index_capacity is not None
assert cap.global_index_capacity is not None
self.assertEqual(cap.total, Throughput(6, 7))
self.assertEqual(cap.table_capacity, Throughput(1, 2))
self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
self.assertEqual(cap.global_index_capacity["g-index"],
Throughput(3, 4))
def test_update_table_throughput(self):
"""Update the table throughput"""
hash_key = DynamoKey("id", data_type=STRING)
self.dynamo.create_table("foobar", hash_key=hash_key, throughput=(1, 1))
tp = Throughput(3, 4)
self.dynamo.update_table("foobar", throughput=tp)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.throughput, tp)
def test_capacity(self):
"""Can return consumed capacity"""
ret = {
"Responses": {
"foo": [],
},
"ConsumedCapacity": [
{
"TableName": "foobar",
"ReadCapacityUnits": 6,
"WriteCapacityUnits": 7,
"Table": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 2,
},
"LocalSecondaryIndexes": {
"l-index": {
"ReadCapacityUnits": 2,
"WriteCapacityUnits": 3,
},
},
"GlobalSecondaryIndexes": {
"g-index": {
"ReadCapacityUnits": 3,
"WriteCapacityUnits": 4,
},
},
}
],
}
with patch.object(self.dynamo.client, "batch_write_item", return_value=ret):
batch = self.dynamo.batch_write("foobar", return_capacity="INDEXES")
with batch:
batch.put({"id": "a"})
cap = batch.consumed_capacity
assert cap is not None
assert cap.table_capacity is not None
assert cap.local_index_capacity is not None
assert cap.global_index_capacity is not None
self.assertEqual(cap.total, Throughput(6, 7))
self.assertEqual(cap.table_capacity, Throughput(1, 2))
self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
self.assertEqual(cap.global_index_capacity["g-index"], Throughput(3, 4))
def test_alter_throughput_from_defaults(self):
""" Updates index throughput values by comparing the current values to those from the actual database.
This might not be a great idea. create_schema doesn't seem to perform any updates with read/write
throughput capacity changes. Not really sure if this should.
"""
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(len(table.global_indexes), 1)
self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))
# Simulating adding an index later on.
WidgetToAddIndex.meta_.global_indexes[
0] = WidgetToAddIndex.meta_.global_indexes[0].throughput(read=10,
write=11)
WidgetToAddIndex.meta_.post_create()
WidgetToAddIndex.meta_.validate_model()
WidgetToAddIndex.meta_.post_validate()
changed = self.engine.update_schema()
self.assertListEqual(
changed,
[WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(table.global_indexes[0].throughput,
Throughput(10, 11))
self.assertEqual(len(table.global_indexes), 1)
one = WidgetToAddIndex("one-1", "one-2", 1)
two = WidgetToAddIndex("one-2", "test", 2)
self.engine.save(one)
self.engine.save(two)
result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
WidgetToAddIndex.string2 == 'test').all()
self.assertEqual(len(result), 1)
self.assertNotEqual(result[0], one)
self.assertEqual(result[0], two)
def test_update_multiple_throughputs(self):
"""Update table and global index throughputs"""
hash_key = DynamoKey("id", data_type=STRING)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field, throughput=(2, 3))
self.dynamo.create_table(
"foobar",
hash_key=hash_key,
global_indexes=[index],
throughput=Throughput(1, 1),
)
tp = Throughput(3, 4)
self.dynamo.update_table(
"foobar",
throughput=tp,
index_updates=[IndexUpdate.update("name-index", tp)],
)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.throughput, tp)
self.assertEqual(table.global_indexes[0].throughput, tp)
def test_clear_all_keep_throughput(self):
""" Calling clear_all will keep same table throughput """
throughput = {}
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.dynamo.update_table(desc.name, Throughput(7, 7))
for index in desc.global_indexes:
self.dynamo.update_table(
desc.name, global_indexes={index.name: Throughput(7, 7)})
self.db.clear_all()
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.assertEqual(desc.throughput.read, 7)
self.assertEqual(desc.throughput.write, 7)
for index in desc.global_indexes:
self.assertEqual(index.throughput.read, 7)
self.assertEqual(index.throughput.write, 7)
def test_create_global_index_throughput(self):
"""Create a table and set throughput on global index"""
hash_key = DynamoKey("id", data_type=STRING)
throughput = Throughput(8, 2)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field, throughput=throughput)
table = Table("foobar", hash_key, global_indexes=[index], throughput=throughput)
self.dynamo.create_table(
"foobar", hash_key=hash_key, global_indexes=[index], throughput=throughput
)
desc = self.dynamo.describe_table("foobar")
self.assertEqual(desc, table)
def test_update_global_index_throughput_old(self):
""" Update throughput on a global index OLD API """
hash_key = DynamoKey('id', data_type=STRING)
index_field = DynamoKey('name')
index = GlobalIndex.all('name-index', index_field)
self.dynamo.create_table('foobar',
hash_key=hash_key,
global_indexes=[index])
tp = Throughput(2, 1)
self.dynamo.update_table('foobar', global_indexes={'name-index': tp})
table = self.dynamo.describe_table('foobar')
self.assertEqual(table.global_indexes[0].throughput, tp)
def test_alter_throughput_directly(self):
""" Tests that throughput provisioning specified directly at update time are applied to the index.
"""
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(len(table.global_indexes), 1)
self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))
changed = self.engine.update_schema(
throughput={
WidgetToAddIndex.meta_.ddb_tablename(): {
"gindex": {
"read": 12,
"write": 13
}
}
})
self.assertListEqual(
changed,
[WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])
table = self.engine.dynamo.describe_table(
WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
self.assertEqual(table.global_indexes[0].throughput,
Throughput(12, 13))
self.assertEqual(len(table.global_indexes), 1)
one = WidgetToAddIndex("one-1", "one-2", 1)
two = WidgetToAddIndex("one-2", "test", 2)
self.engine.save(one)
self.engine.save(two)
result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
WidgetToAddIndex.string2 == 'test').all()
self.assertEqual(len(result), 1)
self.assertNotEqual(result[0], one)
self.assertEqual(result[0], two)
def test_update_index_throughput(self):
"""Update the throughput on a global index"""
hash_key = DynamoKey("id", data_type=STRING)
index_field = DynamoKey("name")
index = GlobalIndex.all("name-index", index_field)
self.dynamo.create_table("foobar", hash_key=hash_key, global_indexes=[index])
tp = Throughput(2, 1)
self.dynamo.update_table(
"foobar", index_updates=[IndexUpdate.update("name-index", tp)]
)
table = self.dynamo.describe_table("foobar")
assert table is not None
self.assertEqual(table.global_indexes[0].throughput, tp)
def test_update_billing_mode(self):
"""Update a table billing mode"""
hash_key = DynamoKey("id", data_type=STRING)
table = self.dynamo.create_table(
"foobar", hash_key=hash_key, billing_mode=PAY_PER_REQUEST
)
assert table is not None
self.assertEqual(table.billing_mode, PAY_PER_REQUEST)
new_table = self.dynamo.update_table(
"foobar", billing_mode=PROVISIONED, throughput=(2, 3)
)
assert new_table is not None
self.assertEqual(new_table.billing_mode, PROVISIONED)
self.assertEqual(new_table.throughput, Throughput(2, 3))
def __init__(self, model):
self.model = model
self._name = model.__name__
self.global_indexes = []
self.orderings = []
self.throughput = Throughput()
self._abstract = False
self.__dict__.update(model.__metadata__)
# Allow throughput to be specified as read/write in a dict
# pylint: disable=E1134
if isinstance(self.throughput, dict):
self.throughput = Throughput(**self.throughput)
# pylint: enable=E1134
self.name = self._name
self.fields = {}
self.hash_key = None
self.range_key = None
self.related_fields = defaultdict(set)
self.all_global_indexes = set()
for gindex in self.global_indexes:
self.all_global_indexes.add(gindex.hash_key)
if gindex.range_key is not None:
self.all_global_indexes.add(gindex.range_key)
def _parse_global_index(self, clause, attrs):
""" Parse a global index clause and return a GlobalIndex """
index_type, name = clause[:2]
name = resolve(name)
def get_key(field, data_type=None):
""" Get or set the DynamoKey from the field name """
if field in attrs:
key = attrs[field]
if data_type is not None:
if TYPES[data_type] != key.data_type:
raise SyntaxError(
"Key %r %s already declared with type %s" % field,
data_type,
key.data_type,
)
else:
if data_type is None:
raise SyntaxError("Missing data type for %r" % field)
key = DynamoKey(field, data_type=TYPES[data_type])
attrs[field] = key
return key
g_hash_key = get_key(*clause.hash_key)
g_range_key = None
# For some reason I can't get the throughput section to have a name
# Use an index instead
tp_index = 3
if clause.range_key:
tp_index += 1
g_range_key = get_key(*clause.range_key)
if clause.include_vars:
tp_index += 1
kwargs = {}
if tp_index < len(clause):
throughput = clause[tp_index]
kwargs["throughput"] = Throughput(*map(resolve, throughput))
index_type = clause.index_type[0]
if index_type in ("ALL", "INDEX"):
factory = GlobalIndex.all
elif index_type == "KEYS":
factory = GlobalIndex.keys
elif index_type == "INCLUDE":
factory = GlobalIndex.include
if not clause.include_vars:
raise SyntaxError("Include index %r missing include fields" %
name)
kwargs["includes"] = [resolve(v) for v in clause.include_vars]
return factory(name, g_hash_key, g_range_key, **kwargs)
def _update_throughput(self, tablename, read, write, index):
"""Update the throughput on a table or index"""
def get_desc() -> Union[TableMeta, GlobalIndexMeta]:
"""Get the table or global index description"""
desc = self.describe(tablename, refresh=True, require=True)
if index is not None:
return desc.global_indexes[index]
return desc
desc = get_desc()
def num_or_star(value):
"""Convert * to -1, otherwise resolve a number"""
return -1 if value == "*" else resolve(value)
read = num_or_star(read)
write = num_or_star(write)
if read < 0:
read = 0 if desc.throughput is None else desc.throughput.read
if write < 0:
write = 0 if desc.throughput is None else desc.throughput.write
throughput = Throughput(read, write)
kwargs = {}
if index:
self.connection.update_table(
tablename, index_updates=[IndexUpdate.update(index, throughput)]
)
elif throughput.read or throughput.write:
self.connection.update_table(
tablename, billing_mode=PROVISIONED, throughput=throughput
)
else:
self.connection.update_table(tablename, billing_mode=PAY_PER_REQUEST)
desc = get_desc()
while desc.status == "UPDATING": # pragma: no cover
time.sleep(5)
desc = get_desc()
def test_throughput_repr(self):
"""Throughput repr should wrap read/write values"""
a = Throughput(1, 1)
self.assertEqual(repr(a), "Throughput(1, 1)")
def test_throughput_eq(self):
"""Throughputs should be equal"""
a, b = Throughput(), Throughput()
self.assertEqual(a, b)
self.assertEqual(hash(a), hash(b))
self.assertFalse(a != b)
def create_dynamo_schema(self,
connection,
tablenames=None,
test=False,
wait=False,
throughput=None,
namespace=()):
"""
Create all Dynamo tables for this model
Parameters
----------
connection : :class:`~dynamo3.DynamoDBConnection`
tablenames : list, optional
List of tables that already exist. Will call 'describe' if not
provided.
test : bool, optional
If True, don't actually create the table (default False)
wait : bool, optional
If True, block until table has been created (default False)
throughput : dict, optional
The throughput of the table and global indexes. Has the keys 'read'
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
namespace : str or tuple, optional
The namespace of the table
Returns
-------
table : str
Table name that was created, or None if nothing created
"""
if self.abstract:
return None
if tablenames is None:
tablenames = set(connection.list_tables())
tablename = self.ddb_tablename(namespace)
if tablename in tablenames:
return None
elif test:
return tablename
indexes = []
global_indexes = []
hash_key = None
if throughput is not None:
table_throughput = Throughput(throughput['read'],
throughput['write'])
else:
table_throughput = self.throughput
hash_key = DynamoKey(self.hash_key.name,
data_type=self.hash_key.ddb_data_type)
range_key = None
if self.range_key is not None:
range_key = DynamoKey(self.range_key.name,
data_type=self.range_key.ddb_data_type)
for field in six.itervalues(self.fields):
if field.index:
idx = field.get_ddb_index()
indexes.append(idx)
for gindex in self.global_indexes:
index = gindex.get_ddb_index(self.fields)
if throughput is not None and gindex.name in throughput:
index.throughput = Throughput(**throughput[gindex.name])
global_indexes.append(index)
if not test:
connection.create_table(tablename, hash_key, range_key, indexes,
global_indexes, table_throughput)
if wait:
desc = connection.describe_table(tablename)
while desc.status != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(tablename)
return tablename
def update_dynamo_schema(self,
connection,
test=False,
wait=False,
throughput=None,
namespace=()):
"""
Updates all Dynamo table global indexes for this model
Parameters
----------
connection : :class:`~dynamo3.DynamoDBConnection`
test : bool, optional
If True, don't actually create the table (default False)
wait : bool, optional
If True, block until table has been created (default False)
throughput : dict, optional
The throughput of the table and global indexes. Has the keys 'read'
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
namespace : str or tuple, optional
The namespace of the table
Returns
-------
table : str
Table name that altered, or None if nothing altered
"""
if self.abstract:
return None
tablename = self.ddb_tablename(namespace)
global_indexes = []
for gindex in self.global_indexes:
index = gindex.get_ddb_index(self.fields)
if throughput is not None and gindex.name in throughput:
index.throughput = Throughput(**throughput[gindex.name])
global_indexes.append(index)
if not global_indexes:
return None
table = connection.describe_table(tablename)
if not table:
return None
expected_indexes = {}
for i in global_indexes:
expected_indexes[i.name] = i
actual_indexes = {}
for i in table.global_indexes:
actual_indexes[i.name] = i
missing_index_names = set(expected_indexes.keys()) - set(
actual_indexes.keys())
missing_indexes = [expected_indexes[i] for i in missing_index_names]
updates = [IndexUpdate.create(index) for index in missing_indexes]
update_indexes_name = set(expected_indexes.keys()) & set(
actual_indexes.keys())
update_indexes = [
expected_indexes[i] for i in update_indexes_name
if actual_indexes[i].throughput != expected_indexes[i].throughput
]
updates.extend([
IndexUpdate.update(index.name, index.throughput)
for index in update_indexes
])
if not updates:
return None
if not test:
connection.update_table(tablename, index_updates=updates)
if wait:
desc = connection.describe_table(tablename)
while desc.status != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(tablename)
return tablename