def test_throttle_total_cap(self):
"""Sleep if consumed capacity exceeds total"""
limiter = RateLimit(total=Capacity(3, 3))
cap = ConsumedCapacity("foobar", Capacity(3, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(1)
def test_throttle_table_default_cap(self):
""" If no table limit provided, use the default """
limiter = RateLimit(default=Capacity(4, 4))
cap = ConsumedCapacity('foobar', Capacity(8, 0), Capacity(8, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_throttle_table_default(self):
"""If no table limit provided, use the default"""
limiter = RateLimit(default_read=4, default_write=4)
cap = ConsumedCapacity("foobar", Capacity(8, 0), Capacity(8, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_count_add_capacity(self):
"""Count addition with consumed_capacity"""
count = Count(4, 2, Capacity(3, 0))
count2 = Count(5, 3, Capacity(2, 0))
ret = count + count2
self.assertEqual(ret, 9)
self.assertEqual(ret.scanned_count, 5)
self.assertEqual(ret.consumed_capacity.read, 5)
def test_throttle_table(self):
""" Sleep if table limit is exceeded """
limiter = RateLimit(3, 3, table_caps={
'foobar': Capacity(0, 4),
})
cap = ConsumedCapacity('foobar', Capacity(8, 0), Capacity(0, 8))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_global_default(self):
""" Global index limit will fall back to table default limit """
limiter = RateLimit(default_read=4, default_write=4)
cap = ConsumedCapacity('foobar',
Capacity(8, 0),
global_index_capacity={
'baz': Capacity(8, 0),
})
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_global_default_table(self):
""" Global index limit defaults to table limit if not present """
limiter = RateLimit(table_caps={
'foobar': Capacity(4, 0),
})
cap = ConsumedCapacity('foobar',
Capacity(8, 0),
global_index_capacity={
'baz': Capacity(8, 0),
})
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_global_index_by_name(self):
""" Global index limit can be specified as tablename:index_name """
limiter = RateLimit(table_caps={
'foobar:baz': Capacity(4, 0),
})
cap = ConsumedCapacity('foobar',
Capacity(8, 0),
global_index_capacity={
'baz': Capacity(8, 0),
})
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_global_index(self):
""" Sleep when global index limit is exceeded """
limiter = RateLimit(table_caps={'foobar': {
'baz': Capacity(4, 0),
}})
cap = ConsumedCapacity('foobar',
Capacity(8, 0),
global_index_capacity={
'baz': Capacity(8, 0),
})
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(2)
def test_local_index(self):
""" Local index capacities count towards the table limit """
limiter = RateLimit(table_caps={
'foobar': Capacity(4, 0),
})
cap = ConsumedCapacity('foobar',
Capacity(8, 0),
local_index_capacity={
'local': Capacity(4, 0),
})
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(1)
def test_global_default(self):
"""Global index limit will fall back to table default limit"""
limiter = RateLimit(default_read=4, default_write=4)
cap = ConsumedCapacity(
"foobar",
Capacity(8, 0),
global_index_capacity={
"baz": Capacity(8, 0),
},
)
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_throttle_table(self):
"""Sleep if table limit is exceeded"""
limiter = RateLimit(
3,
3,
table_caps={
"foobar": Capacity(0, 4),
},
)
cap = ConsumedCapacity("foobar", Capacity(8, 0), Capacity(0, 8))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_global_default_table(self):
"""Global index limit defaults to table limit if not present"""
limiter = RateLimit(table_caps={
"foobar": Capacity(4, 0),
})
cap = ConsumedCapacity(
"foobar",
Capacity(8, 0),
global_index_capacity={
"baz": Capacity(8, 0),
},
)
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_global_index_by_name(self):
"""Global index limit can be specified as tablename:index_name"""
limiter = RateLimit(table_caps={
"foobar:baz": Capacity(4, 0),
})
cap = ConsumedCapacity(
"foobar",
Capacity(8, 0),
global_index_capacity={
"baz": Capacity(8, 0),
},
)
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_global_index(self):
"""Sleep when global index limit is exceeded"""
limiter = RateLimit(table_caps={"foobar": {
"baz": Capacity(4, 0),
}})
cap = ConsumedCapacity(
"foobar",
Capacity(8, 0),
global_index_capacity={
"baz": Capacity(8, 0),
},
)
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_local_index(self):
"""Local index capacities count towards the table limit"""
limiter = RateLimit(table_caps={
"foobar": Capacity(4, 0),
})
cap = ConsumedCapacity(
"foobar",
Capacity(8, 0),
local_index_capacity={
"local": Capacity(4, 0),
},
)
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(1)
def test_throttle_total(self):
""" Sleep if consumed capacity exceeds total """
limiter = RateLimit(3, 3)
cap = ConsumedCapacity('foobar', Capacity(3, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query2('foobar', 'id = :id', id='a'))
sleep.assert_called_with(1)
def test_no_throttle(self):
"""Don't sleep if consumed capacity is within limits"""
limiter = RateLimit(3, 3)
cap = ConsumedCapacity("foobar", Capacity(0, 2))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_not_called()
def test_capacity(self):
""" Can return consumed capacity """
ret = {
'Responses': {
'foo': [],
},
'ConsumedCapacity': [{
'TableName': 'foobar',
'CapacityUnits': 3,
'Table': {
'CapacityUnits': 1,
},
'LocalSecondaryIndexes': {
'l-index': {
'CapacityUnits': 1,
},
},
'GlobalSecondaryIndexes': {
'g-index': {
'CapacityUnits': 1,
},
},
}],
}
with patch.object(self.dynamo.client,
'batch_write_item',
return_value=ret):
batch = self.dynamo.batch_write('foobar',
return_capacity='INDEXES')
with batch:
batch.put({'id': 'a'})
self.assertEqual(batch.consumed_capacity.total, Capacity(0, 3))
def test_throttle_multiply(self):
"""Seconds to sleep is increades to match limit delta"""
limiter = RateLimit(3, 3)
cap = ConsumedCapacity("foobar", Capacity(8, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(3)
def test_throttle_multiple(self):
"""Sleep if the limit is exceeded by multiple calls"""
limiter = RateLimit(4, 4)
cap = ConsumedCapacity("foobar", Capacity(3, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_called_with(2)
def test_count_add_none_capacity(self):
"""Count addition with one None consumed_capacity"""
cap = Capacity(3, 0)
count = Count(4, 2)
count2 = Count(5, 3, cap)
ret = count + count2
self.assertEqual(ret, 9)
self.assertEqual(ret.scanned_count, 5)
self.assertEqual(ret.consumed_capacity, cap)
def test_throttle_callback(self):
"""Callback is called when a query is throttled"""
callback = MagicMock()
callback.return_value = True
limiter = RateLimit(3, 3, callback=callback)
cap = ConsumedCapacity("foobar", Capacity(3, 0))
with self.inject_capacity(cap, limiter) as sleep:
list(self.dynamo.query("foobar", "id = :id", id="a"))
sleep.assert_not_called()
self.assertTrue(callback.called)
def test_capacity_math(self):
""" Capacity addition and equality """
cap = Capacity(2, 4)
s = set([cap])
self.assertIn(Capacity(2, 4), s)
self.assertNotEqual(Capacity(1, 4), cap)
self.assertEqual(Capacity(1, 1) + Capacity(2, 2), Capacity(3, 3))
def test_capacity(self):
"""capacity hooks are called whenever response has ConsumedCapacity"""
hash_key = DynamoKey("id")
self.dynamo.create_table("foobar", hash_key=hash_key)
hook = MagicMock()
self.dynamo.subscribe("capacity", hook)
with patch.object(self.dynamo, "client") as client:
client.scan.return_value = {
"Items": [],
"ConsumedCapacity": {
"TableName": "foobar",
"ReadCapacityUnits": 4,
},
}
rs = self.dynamo.scan("foobar")
list(rs)
cap = ConsumedCapacity("foobar", Capacity(4, 0))
hook.assert_called_with(self.dynamo, "scan", ANY, ANY, cap)
def test_capacity(self):
""" capacity hooks are called whenever response has ConsumedCapacity """
hash_key = DynamoKey('id')
self.dynamo.create_table('foobar', hash_key=hash_key)
hook = MagicMock()
self.dynamo.subscribe('capacity', hook)
with patch.object(self.dynamo, 'client') as client:
client.scan.return_value = {
'Items': [],
'ConsumedCapacity': {
'TableName': 'foobar',
'CapacityUnits': 4,
}
}
rs = self.dynamo.scan('foobar')
list(rs)
cap = ConsumedCapacity('foobar', Capacity(4, 0))
hook.assert_called_with(self.dynamo, 'scan', ANY, ANY, cap)
def test_consumed_capacity_equality(self):
""" ConsumedCapacity addition and equality """
cap = ConsumedCapacity('foobar', Capacity(0, 10), Capacity(0, 2), {
'l-index': Capacity(0, 4),
}, {
'g-index': Capacity(0, 3),
})
c2 = ConsumedCapacity('foobar', Capacity(0, 10), Capacity(0, 2), {
'l-index': Capacity(0, 4),
'l-index2': Capacity(0, 7),
})
self.assertNotEqual(cap, c2)
c3 = ConsumedCapacity('foobar', Capacity(0, 10), Capacity(0, 2), {
'l-index': Capacity(0, 4),
}, {
'g-index': Capacity(0, 3),
})
self.assertIn(cap, set([c3]))
combined = cap + c2
self.assertEqual(
cap + c2,
ConsumedCapacity('foobar', Capacity(0, 20), Capacity(0, 4), {
'l-index': Capacity(0, 8),
'l-index2': Capacity(0, 7),
}, {
'g-index': Capacity(0, 3),
}))
self.assertIn(str(Capacity(0, 3)), str(combined))
def test_consumed_capacity_equality(self):
"""ConsumedCapacity addition and equality"""
cap = ConsumedCapacity(
"foobar",
Capacity(0, 10),
Capacity(0, 2),
{
"l-index": Capacity(0, 4),
},
{
"g-index": Capacity(0, 3),
},
)
c2 = ConsumedCapacity(
"foobar",
Capacity(0, 10),
Capacity(0, 2),
{
"l-index": Capacity(0, 4),
"l-index2": Capacity(0, 7),
},
)
self.assertNotEqual(cap, c2)
c3 = ConsumedCapacity(
"foobar",
Capacity(0, 10),
Capacity(0, 2),
{
"l-index": Capacity(0, 4),
},
{
"g-index": Capacity(0, 3),
},
)
self.assertIn(cap, set([c3]))
combined = cap + c2
self.assertEqual(
cap + c2,
ConsumedCapacity(
"foobar",
Capacity(0, 20),
Capacity(0, 4),
{
"l-index": Capacity(0, 8),
"l-index2": Capacity(0, 7),
},
{
"g-index": Capacity(0, 3),
},
),
)
self.assertIn(str(Capacity(0, 3)), str(combined))
def test_capacity_format(self):
""" String formatting for Capacity """
c = Capacity(1, 3)
self.assertEqual(str(c), "R:1.0 W:3.0")
c = Capacity(0, 0)
self.assertEqual(str(c), "0")
def test_add_different_tables(self):
""" Cannot add ConsumedCapacity of two different tables """
c1 = ConsumedCapacity('foobar', Capacity(1, 28))
c2 = ConsumedCapacity('boofar', Capacity(3, 0))
with self.assertRaises(TypeError):
c1 += c2