Exemplo n.º 1
0
    def _update_throughput(self, tablename, read, write, index):
        """ Update the throughput on a table or index """
        def get_desc():
            """ Get the table or global index description """
            desc = self.describe(tablename, refresh=True, require=True)
            if index is not None:
                return desc.global_indexes[index]
            return desc

        desc = get_desc()

        def num_or_star(value):
            """ Convert * to 0, otherwise resolve a number """
            return 0 if value == "*" else resolve(value)

        read = num_or_star(read)
        write = num_or_star(write)
        if read <= 0:
            read = desc.read_throughput
        if write <= 0:
            write = desc.write_throughput

        throughput = Throughput(read, write)
        kwargs = {}
        if index:
            kwargs["global_indexes"] = {index: throughput}
        else:
            kwargs["throughput"] = throughput
        self.connection.update_table(tablename, **kwargs)
        desc = get_desc()
        while desc.status == "UPDATING":  # pragma: no cover
            time.sleep(5)
            desc = get_desc()
Exemplo n.º 2
0
    def throughput(self, read=5, write=5):
        """
        Set the index throughput

        Parameters
        ----------
        read : int, optional
            Amount of read throughput (default 5)
        write : int, optional
            Amount of write throughput (default 5)

        Notes
        -----
        This is meant to be used as a chain::

            class MyModel(Model):
                __metadata__ = {
                    'global_indexes': [
                        GlobalIndex('myindex', 'hkey', 'rkey').throughput(5, 2)
                    ]
                }

        """
        self._throughput = Throughput(read, write)
        return self
Exemplo n.º 3
0
    def _create(self, tree):
        """ Run a SELECT statement """
        tablename = tree.table
        indexes = []
        global_indexes = []
        hash_key = None
        range_key = None
        attrs = {}
        for declaration in tree.attrs:
            name, type_ = declaration[:2]
            if len(declaration) > 2:
                index = declaration[2]
            else:
                index = None
            if index is not None:
                if index[0] == "HASH":
                    field = hash_key = DynamoKey(name, data_type=TYPES[type_])
                elif index[0] == "RANGE":
                    field = range_key = DynamoKey(name, data_type=TYPES[type_])
                else:
                    index_type = index[0]
                    kwargs = {}
                    if index_type[0] in ("ALL", "INDEX"):
                        factory = LocalIndex.all
                    elif index_type[0] == "KEYS":
                        factory = LocalIndex.keys
                    elif index_type[0] == "INCLUDE":
                        factory = LocalIndex.include
                        kwargs["includes"] = [
                            resolve(v) for v in index.include_vars
                        ]
                    index_name = resolve(index[1])
                    field = DynamoKey(name, data_type=TYPES[type_])
                    idx = factory(index_name, field, **kwargs)
                    indexes.append(idx)
            else:
                field = DynamoKey(name, data_type=TYPES[type_])
            attrs[field.name] = field

        for gindex in tree.global_indexes:
            global_indexes.append(self._parse_global_index(gindex, attrs))

        throughput = None
        if tree.throughput:
            throughput = Throughput(*map(resolve, tree.throughput))

        try:
            ret = self.connection.create_table(
                tablename,
                hash_key,
                range_key,
                indexes=indexes,
                global_indexes=global_indexes,
                throughput=throughput,
            )
        except DynamoDBError as e:
            if e.kwargs["Code"] == "ResourceInUseException" or tree.not_exists:
                return False
            raise
        return True
Exemplo n.º 4
0
 def __init__(self, name, hash_key, range_key=None):
     self.name = name
     self.hash_key = hash_key
     self.range_key = range_key
     self._throughput = Throughput()
     self.ddb_index = GIndex.all
     self.kwargs = {}
Exemplo n.º 5
0
 def test_create_table_throughput(self):
     """Create a table and set throughput"""
     hash_key = DynamoKey("id", data_type=STRING)
     throughput = Throughput(8, 2)
     table = Table("foobar", hash_key, throughput=throughput)
     self.dynamo.create_table("foobar", hash_key=hash_key, throughput=throughput)
     desc = self.dynamo.describe_table("foobar")
     self.assertEqual(desc, table)
Exemplo n.º 6
0
 def test_update_table_throughput(self):
     """ Update the table throughput """
     hash_key = DynamoKey('id', data_type=STRING)
     self.dynamo.create_table('foobar', hash_key=hash_key)
     tp = Throughput(2, 1)
     self.dynamo.update_table('foobar', throughput=tp)
     table = self.dynamo.describe_table('foobar')
     self.assertEqual(table.throughput, tp)
Exemplo n.º 7
0
 def test_capacity(self):
     """Can return consumed capacity"""
     conn = MagicMock()
     response_cap = {
         "TableName": "foobar",
         "ReadCapacityUnits": 6,
         "WriteCapacityUnits": 7,
         "Table": {
             "ReadCapacityUnits": 1,
             "WriteCapacityUnits": 2,
         },
         "LocalSecondaryIndexes": {
             "l-index": {
                 "ReadCapacityUnits": 2,
                 "WriteCapacityUnits": 3,
             },
         },
         "GlobalSecondaryIndexes": {
             "g-index": {
                 "ReadCapacityUnits": 3,
                 "WriteCapacityUnits": 4,
             },
         },
     }
     response = {
         "Responses": [],
         "ConsumedCapacity": [response_cap],
     }
     capacity = ConsumedCapacity.from_response(response_cap)
     response["consumed_capacity"] = [capacity]
     with patch.object(self.dynamo, "client") as client:
         client.transact_get_items.return_value = response
         ret = self.dynamo.txn_get("foobar", [{"id": "a"}])
         list(ret)
     assert ret.consumed_capacity is not None
     cap = ret.consumed_capacity["foobar"]
     assert cap is not None
     assert cap.table_capacity is not None
     assert cap.local_index_capacity is not None
     assert cap.global_index_capacity is not None
     self.assertEqual(cap.total, Throughput(6, 7))
     self.assertEqual(cap.table_capacity, Throughput(1, 2))
     self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
     self.assertEqual(cap.global_index_capacity["g-index"],
                      Throughput(3, 4))
Exemplo n.º 8
0
 def test_update_table_throughput(self):
     """Update the table throughput"""
     hash_key = DynamoKey("id", data_type=STRING)
     self.dynamo.create_table("foobar", hash_key=hash_key, throughput=(1, 1))
     tp = Throughput(3, 4)
     self.dynamo.update_table("foobar", throughput=tp)
     table = self.dynamo.describe_table("foobar")
     assert table is not None
     self.assertEqual(table.throughput, tp)
Exemplo n.º 9
0
 def test_capacity(self):
     """Can return consumed capacity"""
     ret = {
         "Responses": {
             "foo": [],
         },
         "ConsumedCapacity": [
             {
                 "TableName": "foobar",
                 "ReadCapacityUnits": 6,
                 "WriteCapacityUnits": 7,
                 "Table": {
                     "ReadCapacityUnits": 1,
                     "WriteCapacityUnits": 2,
                 },
                 "LocalSecondaryIndexes": {
                     "l-index": {
                         "ReadCapacityUnits": 2,
                         "WriteCapacityUnits": 3,
                     },
                 },
                 "GlobalSecondaryIndexes": {
                     "g-index": {
                         "ReadCapacityUnits": 3,
                         "WriteCapacityUnits": 4,
                     },
                 },
             }
         ],
     }
     with patch.object(self.dynamo.client, "batch_write_item", return_value=ret):
         batch = self.dynamo.batch_write("foobar", return_capacity="INDEXES")
         with batch:
             batch.put({"id": "a"})
     cap = batch.consumed_capacity
     assert cap is not None
     assert cap.table_capacity is not None
     assert cap.local_index_capacity is not None
     assert cap.global_index_capacity is not None
     self.assertEqual(cap.total, Throughput(6, 7))
     self.assertEqual(cap.table_capacity, Throughput(1, 2))
     self.assertEqual(cap.local_index_capacity["l-index"], Throughput(2, 3))
     self.assertEqual(cap.global_index_capacity["g-index"], Throughput(3, 4))
Exemplo n.º 10
0
    def test_alter_throughput_from_defaults(self):
        """ Updates index throughput values by comparing the current values to those from the actual database.

        This might not be a great idea. create_schema doesn't seem to perform any updates with read/write
        throughput capacity changes. Not really sure if this should.
        """
        table = self.engine.dynamo.describe_table(
            WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
        self.assertEqual(len(table.global_indexes), 1)
        self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))

        # Simulating adding an index later on.
        WidgetToAddIndex.meta_.global_indexes[
            0] = WidgetToAddIndex.meta_.global_indexes[0].throughput(read=10,
                                                                     write=11)
        WidgetToAddIndex.meta_.post_create()
        WidgetToAddIndex.meta_.validate_model()
        WidgetToAddIndex.meta_.post_validate()

        changed = self.engine.update_schema()
        self.assertListEqual(
            changed,
            [WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])

        table = self.engine.dynamo.describe_table(
            WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
        self.assertEqual(table.global_indexes[0].throughput,
                         Throughput(10, 11))
        self.assertEqual(len(table.global_indexes), 1)

        one = WidgetToAddIndex("one-1", "one-2", 1)
        two = WidgetToAddIndex("one-2", "test", 2)

        self.engine.save(one)
        self.engine.save(two)

        result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
            WidgetToAddIndex.string2 == 'test').all()

        self.assertEqual(len(result), 1)
        self.assertNotEqual(result[0], one)
        self.assertEqual(result[0], two)
Exemplo n.º 11
0
 def test_update_multiple_throughputs(self):
     """Update table and global index throughputs"""
     hash_key = DynamoKey("id", data_type=STRING)
     index_field = DynamoKey("name")
     index = GlobalIndex.all("name-index", index_field, throughput=(2, 3))
     self.dynamo.create_table(
         "foobar",
         hash_key=hash_key,
         global_indexes=[index],
         throughput=Throughput(1, 1),
     )
     tp = Throughput(3, 4)
     self.dynamo.update_table(
         "foobar",
         throughput=tp,
         index_updates=[IndexUpdate.update("name-index", tp)],
     )
     table = self.dynamo.describe_table("foobar")
     assert table is not None
     self.assertEqual(table.throughput, tp)
     self.assertEqual(table.global_indexes[0].throughput, tp)
Exemplo n.º 12
0
    def test_clear_all_keep_throughput(self):
        """ Calling clear_all will keep same table throughput """
        throughput = {}
        for model in (DynamoPackage, PackageSummary):
            tablename = model.meta_.ddb_tablename(self.engine.namespace)
            desc = self.dynamo.describe_table(tablename)
            self.dynamo.update_table(desc.name, Throughput(7, 7))
            for index in desc.global_indexes:
                self.dynamo.update_table(
                    desc.name, global_indexes={index.name: Throughput(7, 7)})

        self.db.clear_all()

        for model in (DynamoPackage, PackageSummary):
            tablename = model.meta_.ddb_tablename(self.engine.namespace)
            desc = self.dynamo.describe_table(tablename)
            self.assertEqual(desc.throughput.read, 7)
            self.assertEqual(desc.throughput.write, 7)
            for index in desc.global_indexes:
                self.assertEqual(index.throughput.read, 7)
                self.assertEqual(index.throughput.write, 7)
Exemplo n.º 13
0
 def test_create_global_index_throughput(self):
     """Create a table and set throughput on global index"""
     hash_key = DynamoKey("id", data_type=STRING)
     throughput = Throughput(8, 2)
     index_field = DynamoKey("name")
     index = GlobalIndex.all("name-index", index_field, throughput=throughput)
     table = Table("foobar", hash_key, global_indexes=[index], throughput=throughput)
     self.dynamo.create_table(
         "foobar", hash_key=hash_key, global_indexes=[index], throughput=throughput
     )
     desc = self.dynamo.describe_table("foobar")
     self.assertEqual(desc, table)
Exemplo n.º 14
0
 def test_update_global_index_throughput_old(self):
     """ Update throughput on a global index OLD API """
     hash_key = DynamoKey('id', data_type=STRING)
     index_field = DynamoKey('name')
     index = GlobalIndex.all('name-index', index_field)
     self.dynamo.create_table('foobar',
                              hash_key=hash_key,
                              global_indexes=[index])
     tp = Throughput(2, 1)
     self.dynamo.update_table('foobar', global_indexes={'name-index': tp})
     table = self.dynamo.describe_table('foobar')
     self.assertEqual(table.global_indexes[0].throughput, tp)
Exemplo n.º 15
0
    def test_alter_throughput_directly(self):
        """ Tests that throughput provisioning specified directly at update time are applied to the index.

        """
        table = self.engine.dynamo.describe_table(
            WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
        self.assertEqual(len(table.global_indexes), 1)
        self.assertEqual(table.global_indexes[0].throughput, Throughput(5, 5))

        changed = self.engine.update_schema(
            throughput={
                WidgetToAddIndex.meta_.ddb_tablename(): {
                    "gindex": {
                        "read": 12,
                        "write": 13
                    }
                }
            })
        self.assertListEqual(
            changed,
            [WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace)])

        table = self.engine.dynamo.describe_table(
            WidgetToAddIndex.meta_.ddb_tablename(self.engine.namespace))
        self.assertEqual(table.global_indexes[0].throughput,
                         Throughput(12, 13))
        self.assertEqual(len(table.global_indexes), 1)

        one = WidgetToAddIndex("one-1", "one-2", 1)
        two = WidgetToAddIndex("one-2", "test", 2)

        self.engine.save(one)
        self.engine.save(two)

        result = self.engine.query(WidgetToAddIndex).index('gindex').filter(
            WidgetToAddIndex.string2 == 'test').all()

        self.assertEqual(len(result), 1)
        self.assertNotEqual(result[0], one)
        self.assertEqual(result[0], two)
Exemplo n.º 16
0
 def test_update_index_throughput(self):
     """Update the throughput on a global index"""
     hash_key = DynamoKey("id", data_type=STRING)
     index_field = DynamoKey("name")
     index = GlobalIndex.all("name-index", index_field)
     self.dynamo.create_table("foobar", hash_key=hash_key, global_indexes=[index])
     tp = Throughput(2, 1)
     self.dynamo.update_table(
         "foobar", index_updates=[IndexUpdate.update("name-index", tp)]
     )
     table = self.dynamo.describe_table("foobar")
     assert table is not None
     self.assertEqual(table.global_indexes[0].throughput, tp)
Exemplo n.º 17
0
 def test_update_billing_mode(self):
     """Update a table billing mode"""
     hash_key = DynamoKey("id", data_type=STRING)
     table = self.dynamo.create_table(
         "foobar", hash_key=hash_key, billing_mode=PAY_PER_REQUEST
     )
     assert table is not None
     self.assertEqual(table.billing_mode, PAY_PER_REQUEST)
     new_table = self.dynamo.update_table(
         "foobar", billing_mode=PROVISIONED, throughput=(2, 3)
     )
     assert new_table is not None
     self.assertEqual(new_table.billing_mode, PROVISIONED)
     self.assertEqual(new_table.throughput, Throughput(2, 3))
Exemplo n.º 18
0
 def __init__(self, model):
     self.model = model
     self._name = model.__name__
     self.global_indexes = []
     self.orderings = []
     self.throughput = Throughput()
     self._abstract = False
     self.__dict__.update(model.__metadata__)
     # Allow throughput to be specified as read/write in a dict
     # pylint: disable=E1134
     if isinstance(self.throughput, dict):
         self.throughput = Throughput(**self.throughput)
     # pylint: enable=E1134
     self.name = self._name
     self.fields = {}
     self.hash_key = None
     self.range_key = None
     self.related_fields = defaultdict(set)
     self.all_global_indexes = set()
     for gindex in self.global_indexes:
         self.all_global_indexes.add(gindex.hash_key)
         if gindex.range_key is not None:
             self.all_global_indexes.add(gindex.range_key)
Exemplo n.º 19
0
    def _parse_global_index(self, clause, attrs):
        """ Parse a global index clause and return a GlobalIndex """
        index_type, name = clause[:2]
        name = resolve(name)

        def get_key(field, data_type=None):
            """ Get or set the DynamoKey from the field name """
            if field in attrs:
                key = attrs[field]
                if data_type is not None:
                    if TYPES[data_type] != key.data_type:
                        raise SyntaxError(
                            "Key %r %s already declared with type %s" % field,
                            data_type,
                            key.data_type,
                        )
            else:
                if data_type is None:
                    raise SyntaxError("Missing data type for %r" % field)
                key = DynamoKey(field, data_type=TYPES[data_type])
                attrs[field] = key
            return key

        g_hash_key = get_key(*clause.hash_key)
        g_range_key = None
        # For some reason I can't get the throughput section to have a name
        # Use an index instead
        tp_index = 3
        if clause.range_key:
            tp_index += 1
            g_range_key = get_key(*clause.range_key)
        if clause.include_vars:
            tp_index += 1
        kwargs = {}
        if tp_index < len(clause):
            throughput = clause[tp_index]
            kwargs["throughput"] = Throughput(*map(resolve, throughput))
        index_type = clause.index_type[0]
        if index_type in ("ALL", "INDEX"):
            factory = GlobalIndex.all
        elif index_type == "KEYS":
            factory = GlobalIndex.keys
        elif index_type == "INCLUDE":
            factory = GlobalIndex.include
            if not clause.include_vars:
                raise SyntaxError("Include index %r missing include fields" %
                                  name)
            kwargs["includes"] = [resolve(v) for v in clause.include_vars]
        return factory(name, g_hash_key, g_range_key, **kwargs)
Exemplo n.º 20
0
    def _update_throughput(self, tablename, read, write, index):
        """Update the throughput on a table or index"""

        def get_desc() -> Union[TableMeta, GlobalIndexMeta]:
            """Get the table or global index description"""
            desc = self.describe(tablename, refresh=True, require=True)
            if index is not None:
                return desc.global_indexes[index]
            return desc

        desc = get_desc()

        def num_or_star(value):
            """Convert * to -1, otherwise resolve a number"""
            return -1 if value == "*" else resolve(value)

        read = num_or_star(read)
        write = num_or_star(write)
        if read < 0:
            read = 0 if desc.throughput is None else desc.throughput.read
        if write < 0:
            write = 0 if desc.throughput is None else desc.throughput.write

        throughput = Throughput(read, write)

        kwargs = {}
        if index:
            self.connection.update_table(
                tablename, index_updates=[IndexUpdate.update(index, throughput)]
            )
        elif throughput.read or throughput.write:
            self.connection.update_table(
                tablename, billing_mode=PROVISIONED, throughput=throughput
            )
        else:
            self.connection.update_table(tablename, billing_mode=PAY_PER_REQUEST)
        desc = get_desc()
        while desc.status == "UPDATING":  # pragma: no cover
            time.sleep(5)
            desc = get_desc()
Exemplo n.º 21
0
 def test_throughput_repr(self):
     """Throughput repr should wrap read/write values"""
     a = Throughput(1, 1)
     self.assertEqual(repr(a), "Throughput(1, 1)")
Exemplo n.º 22
0
 def test_throughput_eq(self):
     """Throughputs should be equal"""
     a, b = Throughput(), Throughput()
     self.assertEqual(a, b)
     self.assertEqual(hash(a), hash(b))
     self.assertFalse(a != b)
Exemplo n.º 23
0
    def create_dynamo_schema(self,
                             connection,
                             tablenames=None,
                             test=False,
                             wait=False,
                             throughput=None,
                             namespace=()):
        """
        Create all Dynamo tables for this model

        Parameters
        ----------
        connection : :class:`~dynamo3.DynamoDBConnection`
        tablenames : list, optional
            List of tables that already exist. Will call 'describe' if not
            provided.
        test : bool, optional
            If True, don't actually create the table (default False)
        wait : bool, optional
            If True, block until table has been created (default False)
        throughput : dict, optional
            The throughput of the table and global indexes. Has the keys 'read'
            and 'write'. To specify throughput for global indexes, add the name
            of the index as a key and another 'read', 'write' dict as the
            value.
        namespace : str or tuple, optional
            The namespace of the table

        Returns
        -------
        table : str
            Table name that was created, or None if nothing created

        """
        if self.abstract:
            return None
        if tablenames is None:
            tablenames = set(connection.list_tables())
        tablename = self.ddb_tablename(namespace)
        if tablename in tablenames:
            return None
        elif test:
            return tablename

        indexes = []
        global_indexes = []
        hash_key = None

        if throughput is not None:
            table_throughput = Throughput(throughput['read'],
                                          throughput['write'])
        else:
            table_throughput = self.throughput

        hash_key = DynamoKey(self.hash_key.name,
                             data_type=self.hash_key.ddb_data_type)
        range_key = None
        if self.range_key is not None:
            range_key = DynamoKey(self.range_key.name,
                                  data_type=self.range_key.ddb_data_type)
        for field in six.itervalues(self.fields):
            if field.index:
                idx = field.get_ddb_index()
                indexes.append(idx)

        for gindex in self.global_indexes:
            index = gindex.get_ddb_index(self.fields)
            if throughput is not None and gindex.name in throughput:
                index.throughput = Throughput(**throughput[gindex.name])
            global_indexes.append(index)

        if not test:
            connection.create_table(tablename, hash_key, range_key, indexes,
                                    global_indexes, table_throughput)
            if wait:
                desc = connection.describe_table(tablename)
                while desc.status != 'ACTIVE':
                    time.sleep(1)
                    desc = connection.describe_table(tablename)

        return tablename
Exemplo n.º 24
0
    def update_dynamo_schema(self,
                             connection,
                             test=False,
                             wait=False,
                             throughput=None,
                             namespace=()):
        """
        Updates all Dynamo table global indexes for this model

        Parameters
        ----------
        connection : :class:`~dynamo3.DynamoDBConnection`
        test : bool, optional
            If True, don't actually create the table (default False)
        wait : bool, optional
            If True, block until table has been created (default False)
        throughput : dict, optional
            The throughput of the table and global indexes. Has the keys 'read'
            and 'write'. To specify throughput for global indexes, add the name
            of the index as a key and another 'read', 'write' dict as the
            value.
        namespace : str or tuple, optional
            The namespace of the table

        Returns
        -------
        table : str
            Table name that altered, or None if nothing altered

        """
        if self.abstract:
            return None

        tablename = self.ddb_tablename(namespace)

        global_indexes = []

        for gindex in self.global_indexes:
            index = gindex.get_ddb_index(self.fields)
            if throughput is not None and gindex.name in throughput:
                index.throughput = Throughput(**throughput[gindex.name])
            global_indexes.append(index)

        if not global_indexes:
            return None

        table = connection.describe_table(tablename)
        if not table:
            return None

        expected_indexes = {}
        for i in global_indexes:
            expected_indexes[i.name] = i
        actual_indexes = {}
        for i in table.global_indexes:
            actual_indexes[i.name] = i

        missing_index_names = set(expected_indexes.keys()) - set(
            actual_indexes.keys())
        missing_indexes = [expected_indexes[i] for i in missing_index_names]

        updates = [IndexUpdate.create(index) for index in missing_indexes]

        update_indexes_name = set(expected_indexes.keys()) & set(
            actual_indexes.keys())
        update_indexes = [
            expected_indexes[i] for i in update_indexes_name
            if actual_indexes[i].throughput != expected_indexes[i].throughput
        ]

        updates.extend([
            IndexUpdate.update(index.name, index.throughput)
            for index in update_indexes
        ])

        if not updates:
            return None

        if not test:
            connection.update_table(tablename, index_updates=updates)
            if wait:
                desc = connection.describe_table(tablename)
                while desc.status != 'ACTIVE':
                    time.sleep(1)
                    desc = connection.describe_table(tablename)

        return tablename