def test_create_table_throughput(self): """Create a table and set throughput""" hash_key = DynamoKey("id", data_type=STRING) throughput = Throughput(8, 2) table = Table("foobar", hash_key, throughput=throughput) self.dynamo.create_table("foobar", hash_key=hash_key, throughput=throughput) desc = self.dynamo.describe_table("foobar") self.assertEqual(desc, table)
def test_update_table_throughput(self): """ Update the table throughput """ hash_key = DynamoKey('id', data_type=STRING) self.dynamo.create_table('foobar', hash_key=hash_key) tp = Throughput(2, 1) self.dynamo.update_table('foobar', throughput=tp) table = self.dynamo.describe_table('foobar') self.assertEqual(table.throughput, tp)
def test_capacity(self): """Can return consumed capacity""" conn = MagicMock() response_cap = { "TableName": "foobar", "ReadCapacityUnits": 6, "WriteCapacityUnits": 7, "Table": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 2, }, "LocalSecondaryIndexes": { "l-index": { "ReadCapacityUnits": 2, "WriteCapacityUnits": 3, }, }, "GlobalSecondaryIndexes": { "g-index": { "ReadCapacityUnits": 3, "WriteCapacityUnits": 4, }, }, } response = { "Responses": [], "ConsumedCapacity": [response_cap], } capacity = ConsumedCapacity.from_response(response_cap) response["consumed_capacity"] = [capacity] with patch.object(self.dynamo, "client") as client: client.transact_get_items.return_value = response ret = self.dynamo.txn_get("foobar", [{"id": "a"}]) list(ret) assert ret.consumed_capacity is not None cap = ret.consumed_capacity["foobar"] assert cap is not None assert cap.table_capacity is not None assert cap.local_index_capacity is not None assert cap.global_index_capacity is not None self.assertEqual(cap.total, Throughput(6, 7)) self.assertEqual(cap.table_capacity, Throughput(1, 2)) self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3)) self.assertEqual(cap.global_index_capacity["g-index"], Throughput(3, 4))
def test_update_table_throughput(self): """Update the table throughput""" hash_key = DynamoKey("id", data_type=STRING) self.dynamo.create_table("foobar", hash_key=hash_key, throughput=(1, 1)) tp = Throughput(3, 4) self.dynamo.update_table("foobar", throughput=tp) table = self.dynamo.describe_table("foobar") assert table is not None self.assertEqual(table.throughput, tp)
def test_capacity(self): """Can return consumed capacity""" ret = { "Responses": { "foo": [], }, "ConsumedCapacity": [ { "TableName": "foobar", "ReadCapacityUnits": 6, "WriteCapacityUnits": 7, "Table": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 2, }, "LocalSecondaryIndexes": { "l-index": { "ReadCapacityUnits": 2, "WriteCapacityUnits": 3, }, }, "GlobalSecondaryIndexes": { "g-index": { "ReadCapacityUnits": 3, "WriteCapacityUnits": 4, }, }, } ], } with patch.object(self.dynamo.client, "batch_write_item", return_value=ret): batch = self.dynamo.batch_write("foobar", return_capacity="INDEXES") with batch: batch.put({"id": "a"}) cap = batch.consumed_capacity assert cap is not None assert cap.table_capacity is not None assert cap.local_index_capacity is not None assert cap.global_index_capacity is not None self.assertEqual(cap.total, Throughput(6, 7)) self.assertEqual(cap.table_capacity, Throughput(1, 2)) self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3)) self.assertEqual(cap.global_index_capacity["g-index"], Throughput(3, 4))
def test_alter_throughput_from_defaults(self): """ Updates index throughput values by comparing the current values to those from the actual database. This might not be a great idea. create_schema doesn't seem to perform any updates with read/write throughput capacity changes. Not really sure if this should. """ table = self.engine.dynamo.describe_table( WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)) self.assertEqual(len(table.global_indexes), 1) self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5)) # Simulating adding an index later on. WidgetToAddIndex.meta_.global_indexes[ 0] = WidgetToAddIndex.meta_.global_indexes[0].throughput(read=10, write=11) WidgetToAddIndex.meta_.post_create() WidgetToAddIndex.meta_.validate_model() WidgetToAddIndex.meta_.post_validate() changed = self.engine.update_schema() self.assertListEqual( changed, [WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)]) table = self.engine.dynamo.describe_table( WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)) self.assertEqual(table.global_indexes[0].throughput, Throughput(10, 11)) self.assertEqual(len(table.global_indexes), 1) one = WidgetToAddIndex("one-1", "one-2", 1) two = WidgetToAddIndex("one-2", "test", 2) self.engine.save(one) self.engine.save(two) result = self.engine.query(WidgetToAddIndex).index('gindex').filter( WidgetToAddIndex.string2 == 'test').all() self.assertEqual(len(result), 1) self.assertNotEqual(result[0], one) self.assertEqual(result[0], two)
def test_update_global_index_throughput_old(self): """ Update throughput on a global index OLD API """ hash_key = DynamoKey('id', data_type=STRING) index_field = DynamoKey('name') index = GlobalIndex.all('name-index', index_field) self.dynamo.create_table('foobar', hash_key=hash_key, global_indexes=[index]) tp = Throughput(2, 1) self.dynamo.update_table('foobar', global_indexes={'name-index': tp}) table = self.dynamo.describe_table('foobar') self.assertEqual(table.global_indexes[0].throughput, tp)
def test_update_multiple_throughputs(self): """Update table and global index throughputs""" hash_key = DynamoKey("id", data_type=STRING) index_field = DynamoKey("name") index = GlobalIndex.all("name-index", index_field, throughput=(2, 3)) self.dynamo.create_table( "foobar", hash_key=hash_key, global_indexes=[index], throughput=Throughput(1, 1), ) tp = Throughput(3, 4) self.dynamo.update_table( "foobar", throughput=tp, index_updates=[IndexUpdate.update("name-index", tp)], ) table = self.dynamo.describe_table("foobar") assert table is not None self.assertEqual(table.throughput, tp) self.assertEqual(table.global_indexes[0].throughput, tp)
def test_create_global_index_throughput(self): """Create a table and set throughput on global index""" hash_key = DynamoKey("id", data_type=STRING) throughput = Throughput(8, 2) index_field = DynamoKey("name") index = GlobalIndex.all("name-index", index_field, throughput=throughput) table = Table("foobar", hash_key, global_indexes=[index], throughput=throughput) self.dynamo.create_table( "foobar", hash_key=hash_key, global_indexes=[index], throughput=throughput ) desc = self.dynamo.describe_table("foobar") self.assertEqual(desc, table)
def test_alter_throughput_directly(self): """ Tests that throughput provisioning specified directly at update time are applied to the index. """ table = self.engine.dynamo.describe_table( WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)) self.assertEqual(len(table.global_indexes), 1) self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5)) changed = self.engine.update_schema( throughput={ WidgetToAddIndex.meta_.ddb_tablename(): { "gindex": { "read": 12, "write": 13 } } }) self.assertListEqual( changed, [WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)]) table = self.engine.dynamo.describe_table( WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)) self.assertEqual(table.global_indexes[0].throughput, Throughput(12, 13)) self.assertEqual(len(table.global_indexes), 1) one = WidgetToAddIndex("one-1", "one-2", 1) two = WidgetToAddIndex("one-2", "test", 2) self.engine.save(one) self.engine.save(two) result = self.engine.query(WidgetToAddIndex).index('gindex').filter( WidgetToAddIndex.string2 == 'test').all() self.assertEqual(len(result), 1) self.assertNotEqual(result[0], one) self.assertEqual(result[0], two)
def test_update_index_throughput(self): """Update the throughput on a global index""" hash_key = DynamoKey("id", data_type=STRING) index_field = DynamoKey("name") index = GlobalIndex.all("name-index", index_field) self.dynamo.create_table("foobar", hash_key=hash_key, global_indexes=[index]) tp = Throughput(2, 1) self.dynamo.update_table( "foobar", index_updates=[IndexUpdate.update("name-index", tp)] ) table = self.dynamo.describe_table("foobar") assert table is not None self.assertEqual(table.global_indexes[0].throughput, tp)
def __init__(self, model): self.model = model self._name = model.__name__ self.global_indexes = [] self.orderings = [] self.throughput = Throughput() self._abstract = False self.__dict__.update(model.__metadata__) # Allow throughput to be specified as read/write in a dict # pylint: disable=E1134 if isinstance(self.throughput, dict): self.throughput = Throughput(**self.throughput) # pylint: enable=E1134 self.name = self._name self.fields = {} self.hash_key = None self.range_key = None self.related_fields = defaultdict(set) self.all_global_indexes = set() for gindex in self.global_indexes: self.all_global_indexes.add(gindex.hash_key) if gindex.range_key is not None: self.all_global_indexes.add(gindex.range_key)
def test_update_billing_mode(self): """Update a table billing mode""" hash_key = DynamoKey("id", data_type=STRING) table = self.dynamo.create_table( "foobar", hash_key=hash_key, billing_mode=PAY_PER_REQUEST ) assert table is not None self.assertEqual(table.billing_mode, PAY_PER_REQUEST) new_table = self.dynamo.update_table( "foobar", billing_mode=PROVISIONED, throughput=(2, 3) ) assert new_table is not None self.assertEqual(new_table.billing_mode, PROVISIONED) self.assertEqual(new_table.throughput, Throughput(2, 3))
def update_dynamo_schema(self, connection, test=False, wait=False, throughput=None, namespace=()): """ Updates all Dynamo table global indexes for this model Parameters ---------- connection : :class:`~dynamo3.DynamoDBConnection` test : bool, optional If True, don't actually create the table (default False) wait : bool, optional If True, block until table has been created (default False) throughput : dict, optional The throughput of the table and global indexes. Has the keys 'read' and 'write'. To specify throughput for global indexes, add the name of the index as a key and another 'read', 'write' dict as the value. namespace : str or tuple, optional The namespace of the table Returns ------- table : str Table name that altered, or None if nothing altered """ if self.abstract: return None tablename = self.ddb_tablename(namespace) global_indexes = [] for gindex in self.global_indexes: index = gindex.get_ddb_index(self.fields) if throughput is not None and gindex.name in throughput: index.throughput = Throughput(**throughput[gindex.name]) global_indexes.append(index) if not global_indexes: return None table = connection.describe_table(tablename) if not table: return None expected_indexes = {} for i in global_indexes: expected_indexes[i.name] = i actual_indexes = {} for i in table.global_indexes: actual_indexes[i.name] = i missing_index_names = set(expected_indexes.keys()) - set( actual_indexes.keys()) missing_indexes = [expected_indexes[i] for i in missing_index_names] updates = [IndexUpdate.create(index) for index in missing_indexes] update_indexes_name = set(expected_indexes.keys()) & set( actual_indexes.keys()) update_indexes = [ expected_indexes[i] for i in update_indexes_name if actual_indexes[i].throughput != expected_indexes[i].throughput ] updates.extend([ IndexUpdate.update(index.name, index.throughput) for index in update_indexes ]) if not updates: return None if not test: connection.update_table(tablename, index_updates=updates) if wait: desc = connection.describe_table(tablename) while desc.status != 'ACTIVE': time.sleep(1) desc = connection.describe_table(tablename) return tablename
def create_dynamo_schema(self, connection, tablenames=None, test=False, wait=False, throughput=None, namespace=()): """ Create all Dynamo tables for this model Parameters ---------- connection : :class:`~dynamo3.DynamoDBConnection` tablenames : list, optional List of tables that already exist. Will call 'describe' if not provided. test : bool, optional If True, don't actually create the table (default False) wait : bool, optional If True, block until table has been created (default False) throughput : dict, optional The throughput of the table and global indexes. Has the keys 'read' and 'write'. To specify throughput for global indexes, add the name of the index as a key and another 'read', 'write' dict as the value. namespace : str or tuple, optional The namespace of the table Returns ------- table : str Table name that was created, or None if nothing created """ if self.abstract: return None if tablenames is None: tablenames = set(connection.list_tables()) tablename = self.ddb_tablename(namespace) if tablename in tablenames: return None elif test: return tablename indexes = [] global_indexes = [] hash_key = None if throughput is not None: table_throughput = Throughput(throughput['read'], throughput['write']) else: table_throughput = self.throughput hash_key = DynamoKey(self.hash_key.name, data_type=self.hash_key.ddb_data_type) range_key = None if self.range_key is not None: range_key = DynamoKey(self.range_key.name, data_type=self.range_key.ddb_data_type) for field in six.itervalues(self.fields): if field.index: idx = field.get_ddb_index() indexes.append(idx) for gindex in self.global_indexes: index = gindex.get_ddb_index(self.fields) if throughput is not None and gindex.name in throughput: index.throughput = Throughput(**throughput[gindex.name]) global_indexes.append(index) if not test: connection.create_table(tablename, hash_key, range_key, indexes, global_indexes, table_throughput) if wait: desc = connection.describe_table(tablename) while desc.status != 'ACTIVE': time.sleep(1) desc = connection.describe_table(tablename) return tablename