def test_ttl_expiration_hash_wrong_type(dynamodb):
duration = 900 if is_aws(dynamodb) else 3
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
],
AttributeDefinitions=[{
'AttributeName': 'p',
'AttributeType': 'S'
}]) as table:
ttl_spec = {'AttributeName': 'p', 'Enabled': True}
table.meta.client.update_time_to_live(TableName=table.name,
TimeToLiveSpecification=ttl_spec)
# p1 is in the past, but it's a string, not the required numeric type,
# so the item should never expire.
p1 = str(int(time.time()) - 60)
table.put_item(Item={'p': p1})
start_time = time.time()
while time.time() < start_time + duration:
print(f"--- {int(time.time()-start_time)} seconds")
if 'Item' in table.get_item(Key={'p': p1}):
print("p1 alive")
time.sleep(duration / 15)
# After the delay, p2 should be alive, p1 should not
assert 'Item' in table.get_item(Key={'p': p1})
def test_ttl_expiration_long(dynamodb):
# Write 100*N items to the table, 1/100th of them are expired. The test
# completes when all of them are expired (or on timeout).
# To compilicate matter for the paging that we're trying to test,
# have N partitions with 100 items in each, so potentially the paging
# might stop in the middle of a partition.
# We need to pick a relatively big N to cause the 100*N items to exceed
# the size of a single page (I tested this by stopping the scanner after
# the first page and checking which N starts to generate incorrect results)
N = 400
max_duration = 1200 if is_aws(dynamodb) else 15
with new_test_table(dynamodb,
KeySchema=[{
'AttributeName': 'p',
'KeyType': 'HASH'
}, {
'AttributeName': 'c',
'KeyType': 'RANGE'
}],
AttributeDefinitions=[{
'AttributeName': 'p',
'AttributeType': 'N'
}, {
'AttributeName': 'c',
'AttributeType': 'N'
}]) as table:
ttl_spec = {'AttributeName': 'expiration', 'Enabled': True}
response = table.meta.client.update_time_to_live(
TableName=table.name, TimeToLiveSpecification=ttl_spec)
with table.batch_writer() as batch:
for p in range(N):
for c in range(100):
# Only the first item (c==0) in each partition will expire
expiration = int(time.time()) - 60 if c == 0 else int(
time.time()) + 3600
batch.put_item(Item={
'p': p,
'c': c,
'expiration': expiration
})
start_time = time.time()
while time.time() < start_time + max_duration:
# We could have used Select=COUNT here, but Alternator doesn't
# implement it yet (issue #5058).
count = 0
response = table.scan(ConsistentRead=True)
if 'Count' in response:
count += response['Count']
while 'LastEvaluatedKey' in response:
response = table.scan(
ExclusiveStartKey=response['LastEvaluatedKey'],
ConsistentRead=True)
if 'Count' in response:
count += response['Count']
print(count)
if count == 99 * N:
break
time.sleep(max_duration / 15.0)
assert count == 99 * N
def test_ttl_expiration_with_rangekey(dynamodb):
# Note that unlike test_ttl_expiration, this test doesn't have a fixed
# duration - it finishes as soon as the item we expect to be expired
# is expired.
max_duration = 1200 if is_aws(dynamodb) else 10
with new_test_table(dynamodb,
KeySchema=[{
'AttributeName': 'p',
'KeyType': 'HASH'
}, {
'AttributeName': 'c',
'KeyType': 'RANGE'
}],
AttributeDefinitions=[{
'AttributeName': 'p',
'AttributeType': 'S'
}, {
'AttributeName': 'c',
'AttributeType': 'S'
}]) as table:
# Enable TTL, using the attribute "expiration":
client = table.meta.client
ttl_spec = {'AttributeName': 'expiration', 'Enabled': True}
response = client.update_time_to_live(TableName=table.name,
TimeToLiveSpecification=ttl_spec)
assert response['TimeToLiveSpecification'] == ttl_spec
# This item should never expire, it is missing the "expiration"
# attribute:
p1 = random_string()
c1 = random_string()
table.put_item(Item={'p': p1, 'c': c1})
# This item should expire ASAP, as its "expiration" has already
# passed, one minute ago:
p2 = random_string()
c2 = random_string()
table.put_item(Item={
'p': p2,
'c': c2,
'expiration': int(time.time()) - 60
})
start_time = time.time()
while time.time() < start_time + max_duration:
if ('Item' in table.get_item(Key={
'p': p1,
'c': c1
})) and not ('Item' in table.get_item(Key={
'p': p2,
'c': c2
})):
# p2 expired, p1 didn't. We're done
break
time.sleep(max_duration / 15.0)
assert 'Item' in table.get_item(Key={'p': p1, 'c': c1})
assert not 'Item' in table.get_item(Key={'p': p2, 'c': c2})
def check_pre_raft(dynamodb):
from util import is_aws
# If not running on Scylla, return false.
if is_aws(dynamodb):
return false
# In Scylla, we check Raft mode by inspecting the configuration via a
# system table (which is also visible in Alternator)
config_table = dynamodb.Table('.scylla.alternator.system.config')
experimental_features = config_table.query(
KeyConditionExpression='#key=:val',
ExpressionAttributeNames={'#key': 'name'},
ExpressionAttributeValues={':val': 'experimental_features'
})['Items'][0]['value']
return not '"raft"' in experimental_features
def test_ttl_expiration_range(dynamodb):
duration = 1200 if is_aws(dynamodb) else 10
with new_test_table(dynamodb,
KeySchema=[{
'AttributeName': 'p',
'KeyType': 'HASH'
}, {
'AttributeName': 'c',
'KeyType': 'RANGE'
}],
AttributeDefinitions=[{
'AttributeName': 'p',
'AttributeType': 'S'
}, {
'AttributeName': 'c',
'AttributeType': 'N'
}]) as table:
ttl_spec = {'AttributeName': 'c', 'Enabled': True}
table.meta.client.update_time_to_live(TableName=table.name,
TimeToLiveSpecification=ttl_spec)
# c1 is in the past, and should be expired. c2 is in the distant
# future and should not be expired.
p = random_string()
c1 = int(time.time()) - 60
c2 = int(time.time()) + 3600
table.put_item(Item={'p': p, 'c': c1})
table.put_item(Item={'p': p, 'c': c2})
start_time = time.time()
def check_expired():
return not 'Item' in table.get_item(Key={
'p': p,
'c': c1
}) and 'Item' in table.get_item(Key={
'p': p,
'c': c2
})
while time.time() < start_time + duration:
print(f"--- {int(time.time()-start_time)} seconds")
if 'Item' in table.get_item(Key={'p': p, 'c': c1}):
print("c1 alive")
if 'Item' in table.get_item(Key={'p': p, 'c': c2}):
print("c2 alive")
if check_expired():
break
time.sleep(duration / 15)
# After the delay, c2 should be alive, c1 should not
assert check_expired()
def rest_api(dynamodb):
if is_aws(dynamodb):
pytest.skip('Scylla-only REST API not supported by AWS')
url = dynamodb.meta.client._endpoint.host
# The REST API is on port 10000, and always http, not https.
url = re.sub(r':[0-9]+(/|$)', ':10000', url)
url = re.sub(r'^https:', 'http:', url)
# Scylla's REST API does not have an official "ping" command,
# so we just list the keyspaces as a (usually) short operation
try:
requests.get(f'{url}/column_family/name/keyspace',
timeout=1).raise_for_status()
except:
pytest.skip('Cannot connect to Scylla REST API')
return url
def scylla_only(dynamodb):
if is_aws(dynamodb):
pytest.skip('Scylla-only feature not supported by AWS')
def test_ttl_expiration_streams(dynamodb, dynamodbstreams):
# In my experiments, a 30-minute (1800 seconds) is the typical
# expiration delay in this test. If the test doesn't finish within
# max_duration, we report a failure.
max_duration = 3600 if is_aws(dynamodb) else 10
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
{
'AttributeName': 'c',
'KeyType': 'RANGE'
},
],
AttributeDefinitions=[
{
'AttributeName': 'p',
'AttributeType': 'S'
},
{
'AttributeName': 'c',
'AttributeType': 'S'
},
],
StreamSpecification={
'StreamEnabled': True,
'StreamViewType': 'NEW_AND_OLD_IMAGES'
}) as table:
ttl_spec = {'AttributeName': 'expiration', 'Enabled': True}
table.meta.client.update_time_to_live(TableName=table.name,
TimeToLiveSpecification=ttl_spec)
# Before writing to the table, wait for the stream to become active
# so we can be sure that the expiration - even if it miraculously
# happens in a second (it usually takes 30 minutes) - is guaranteed
# to reach the stream.
stream_enabled = False
start_time = time.time()
while time.time() < start_time + 120:
desc = table.meta.client.describe_table(
TableName=table.name)['Table']
if 'LatestStreamArn' in desc:
arn = desc['LatestStreamArn']
desc = dynamodbstreams.describe_stream(StreamArn=arn)
if desc['StreamDescription']['StreamStatus'] == 'ENABLED':
stream_enabled = True
break
time.sleep(10)
assert stream_enabled
# Write a single expiring item. Set its expiration one minute in the
# past, so the item should expire ASAP.
p = random_string()
c = random_string()
expiration = int(time.time()) - 60
table.put_item(Item={
'p': p,
'c': c,
'animal': 'dog',
'expiration': expiration
})
# Wait (up to max_duration) for the item to expire, and for the
# expiration event to reach the stream:
start_time = time.time()
event_found = False
while time.time() < start_time + max_duration:
print(f"--- {int(time.time()-start_time)} seconds")
item_expired = not 'Item' in table.get_item(Key={'p': p, 'c': c})
for record in read_entire_stream(dynamodbstreams, table):
# An expired item has a specific userIdentity as follows:
if record.get('userIdentity') == {
'Type': 'Service',
'PrincipalId': 'dynamodb.amazonaws.com'
}:
# The expired item should be a REMOVE, and because we
# asked for old images and both the key and the full
# content.
assert record['eventName'] == 'REMOVE'
assert record['dynamodb']['Keys'] == {
'p': {
'S': p
},
'c': {
'S': c
}
}
assert record['dynamodb']['OldImage'] == {
'p': {
'S': p
},
'c': {
'S': c
},
'animal': {
'S': 'dog'
},
'expiration': {
'N': str(expiration)
}
}
event_found = True
print(f"item expired {item_expired} event {event_found}")
if item_expired and event_found:
return
time.sleep(max_duration / 15)
pytest.fail('item did not expire or event did not reach stream')
def test_ttl_expiration_lsi_key(dynamodb):
# In my experiments, a 30-minute (1800 seconds) is the typical delay
# for expiration in this test for DynamoDB
max_duration = 3600 if is_aws(dynamodb) else 10
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
{
'AttributeName': 'c',
'KeyType': 'RANGE'
},
],
LocalSecondaryIndexes=[
{
'IndexName':
'lsi',
'KeySchema': [
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
{
'AttributeName': 'l',
'KeyType': 'RANGE'
},
],
'Projection': {
'ProjectionType': 'ALL'
},
},
],
AttributeDefinitions=[
{
'AttributeName': 'p',
'AttributeType': 'S'
},
{
'AttributeName': 'c',
'AttributeType': 'S'
},
{
'AttributeName': 'l',
'AttributeType': 'N'
},
]) as table:
# The expiration-time attribute is the LSI key "l":
ttl_spec = {'AttributeName': 'l', 'Enabled': True}
response = table.meta.client.update_time_to_live(
TableName=table.name, TimeToLiveSpecification=ttl_spec)
assert 'TimeToLiveSpecification' in response
assert response['TimeToLiveSpecification'] == ttl_spec
p = random_string()
c = random_string()
l = random_string()
# expiration one minute in the past, so item should expire ASAP.
expiration = int(time.time()) - 60
table.put_item(Item={'p': p, 'c': c, 'l': expiration})
start_time = time.time()
gsi_was_alive = False
while time.time() < start_time + max_duration:
print(f"--- {int(time.time()-start_time)} seconds")
if 'Item' in table.get_item(Key={'p': p, 'c': c}):
print("base alive")
else:
# test is done - and successful:
return
time.sleep(max_duration / 15)
pytest.fail('base not expired')
def test_ttl_expiration_gsi_lsi(dynamodb):
# In our experiments we noticed that when a table has secondary indexes,
# items are expired with significant delay. Whereas a 10 minute delay
# for regular tables was typical, in the table we have here we saw
# a typical delay of 30 minutes before items expired.
max_duration = 3600 if is_aws(dynamodb) else 10
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
{
'AttributeName': 'c',
'KeyType': 'RANGE'
},
],
LocalSecondaryIndexes=[
{
'IndexName':
'lsi',
'KeySchema': [
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
{
'AttributeName': 'l',
'KeyType': 'RANGE'
},
],
'Projection': {
'ProjectionType': 'ALL'
},
},
],
GlobalSecondaryIndexes=[
{
'IndexName':
'gsi',
'KeySchema': [
{
'AttributeName': 'g',
'KeyType': 'HASH'
},
],
'Projection': {
'ProjectionType': 'ALL'
}
},
],
AttributeDefinitions=[
{
'AttributeName': 'p',
'AttributeType': 'S'
},
{
'AttributeName': 'c',
'AttributeType': 'S'
},
{
'AttributeName': 'g',
'AttributeType': 'S'
},
{
'AttributeName': 'l',
'AttributeType': 'S'
},
]) as table:
ttl_spec = {'AttributeName': 'expiration', 'Enabled': True}
response = table.meta.client.update_time_to_live(
TableName=table.name, TimeToLiveSpecification=ttl_spec)
assert 'TimeToLiveSpecification' in response
assert response['TimeToLiveSpecification'] == ttl_spec
p = random_string()
c = random_string()
g = random_string()
l = random_string()
# expiration one minute in the past, so item should expire ASAP.
expiration = int(time.time()) - 60
table.put_item(Item={
'p': p,
'c': c,
'g': g,
'l': l,
'expiration': expiration
})
start_time = time.time()
gsi_was_alive = False
while time.time() < start_time + max_duration:
print(f"--- {int(time.time()-start_time)} seconds")
base_alive = 'Item' in table.get_item(Key={'p': p, 'c': c})
gsi_alive = bool(
full_query(table,
IndexName='gsi',
ConsistentRead=False,
KeyConditionExpression="g=:g",
ExpressionAttributeValues={':g': g}))
lsi_alive = bool(
full_query(table,
IndexName='lsi',
KeyConditionExpression="p=:p and l=:l",
ExpressionAttributeValues={
':p': p,
':l': l
}))
if base_alive:
print("base alive")
if gsi_alive:
print("gsi alive")
# gsi takes time to go up, so make sure it did
gsi_was_alive = True
if lsi_alive:
print("lsi alive")
# If the base item, gsi item and lsi item have all expired, the
# test is done - and successful:
if not base_alive and not gsi_alive and gsi_was_alive and not lsi_alive:
return
time.sleep(max_duration / 15)
pytest.fail('base, gsi, or lsi not expired')
def test_ttl_expiration(dynamodb):
duration = 1200 if is_aws(dynamodb) else 10
# delta is a quarter of the test duration, but no less than one second,
# and we use it to schedule some expirations a bit after the test starts,
# not immediately.
delta = math.ceil(duration / 4)
assert delta >= 1
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'p',
'KeyType': 'HASH'
},
],
AttributeDefinitions=[{
'AttributeName': 'p',
'AttributeType': 'S'
}]) as table:
# Insert one expiring item *before* enabling the TTL, to verify that
# items that already exist when TTL is configured also get handled.
p0 = random_string()
table.put_item(Item={'p': p0, 'expiration': int(time.time()) - 60})
# Enable TTL, using the attribute "expiration":
client = table.meta.client
ttl_spec = {'AttributeName': 'expiration', 'Enabled': True}
response = client.update_time_to_live(TableName=table.name,
TimeToLiveSpecification=ttl_spec)
assert response['TimeToLiveSpecification'] == ttl_spec
# This item should never expire, it is missing the "expiration"
# attribute:
p1 = random_string()
table.put_item(Item={'p': p1})
# This item should expire ASAP, as its "expiration" has already
# passed, one minute ago:
p2 = random_string()
table.put_item(Item={'p': p2, 'expiration': int(time.time()) - 60})
# This item has an expiration more than 5 years in the past (it is my
# birth date...), so according to the DynamoDB documentation it should
# be ignored and p3 should never expire:
p3 = random_string()
table.put_item(Item={'p': p3, 'expiration': 162777600})
# This item has as its expiration delta into the future, which is a
# small part of the test duration, so should expire by the time the
# test ends:
p4 = random_string()
table.put_item(Item={'p': p4, 'expiration': int(time.time()) + delta})
# This item starts with expiration time delta into the future,
# but before it expires we will move it again, so it will never expire:
p5 = random_string()
table.put_item(Item={'p': p5, 'expiration': int(time.time()) + delta})
# This item has an expiration time two durations into the future, so it
# will not expire by the time the test ends:
p6 = random_string()
table.put_item(Item={
'p': p6,
'expiration': int(time.time() + duration * 2)
})
# Like p4, this item has an expiration time delta into the future,
# here the expiration time is wrongly encoded as a string, not a
# number, so the item should never expire:
p7 = random_string()
table.put_item(Item={
'p': p7,
'expiration': str(int(time.time()) + delta)
})
# Like p2, p8 and p9 also have an already-passed expiration time,
# and should expire ASAP. However, whereas p2 had a straighforward
# integer like 12345678 as the expiration time, p8 and p9 have
# slightly more elaborate numbers: p8 has 1234567e1 and p9 has
# 12345678.1234. Those formats should be fine, and this test verifies
# the TTL code's number parsing doesn't get confused (in our original
# implementation, it did).
p8 = random_string()
with client_no_transform(table.meta.client) as client:
client.put_item(TableName=table.name,
Item={
'p': {
'S': p8
},
'expiration': {
'N': str(
(int(time.time()) - 60) // 10) + "e1"
}
})
# Similarly, floating point expiration time like 12345678.1 should
# also be fine (note that Python's time.time() returns floating point).
# This item should also be expired ASAP too.
p9 = random_string()
print(Decimal(str(time.time() - 60)))
table.put_item(Item={
'p': p9,
'expiration': Decimal(str(time.time() - 60))
})
def check_expired():
# After the delay, p1,p3,p5,p6,p7 should be alive, p0,p2,p4 should not
return not 'Item' in table.get_item(Key={'p': p0}) \
and 'Item' in table.get_item(Key={'p': p1}) \
and not 'Item' in table.get_item(Key={'p': p2}) \
and 'Item' in table.get_item(Key={'p': p3}) \
and not 'Item' in table.get_item(Key={'p': p4}) \
and 'Item' in table.get_item(Key={'p': p5}) \
and 'Item' in table.get_item(Key={'p': p6}) \
and 'Item' in table.get_item(Key={'p': p7}) \
and not 'Item' in table.get_item(Key={'p': p8}) \
and not 'Item' in table.get_item(Key={'p': p9})
# We could have just done time.sleep(duration) here, but in case a
# user is watching this long test, let's output the status every
# minute, and it also allows us to test what happens when an item
# p5's expiration time is continuously pushed back into the future:
start_time = time.time()
while time.time() < start_time + duration:
print(f"--- {int(time.time()-start_time)} seconds")
if 'Item' in table.get_item(Key={'p': p0}):
print("p0 alive")
if 'Item' in table.get_item(Key={'p': p1}):
print("p1 alive")
if 'Item' in table.get_item(Key={'p': p2}):
print("p2 alive")
if 'Item' in table.get_item(Key={'p': p3}):
print("p3 alive")
if 'Item' in table.get_item(Key={'p': p4}):
print("p4 alive")
if 'Item' in table.get_item(Key={'p': p5}):
print("p5 alive")
if 'Item' in table.get_item(Key={'p': p6}):
print("p6 alive")
if 'Item' in table.get_item(Key={'p': p7}):
print("p7 alive")
if 'Item' in table.get_item(Key={'p': p8}):
print("p8 alive")
if 'Item' in table.get_item(Key={'p': p9}):
print("p9 alive")
# Always keep p5's expiration delta into the future
table.update_item(Key={'p': p5},
AttributeUpdates={
'expiration': {
'Value': int(time.time()) + delta,
'Action': 'PUT'
}
})
if check_expired():
break
time.sleep(duration / 15.0)
assert check_expired()
