def test_ttl_expiration_hash_wrong_type(dynamodb): duration = 900 if is_aws(dynamodb) else 3 with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, ], AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }]) as table: ttl_spec = {'AttributeName': 'p', 'Enabled': True} table.meta.client.update_time_to_live(TableName=table.name, TimeToLiveSpecification=ttl_spec) # p1 is in the past, but it's a string, not the required numeric type, # so the item should never expire. p1 = str(int(time.time()) - 60) table.put_item(Item={'p': p1}) start_time = time.time() while time.time() < start_time + duration: print(f"--- {int(time.time()-start_time)} seconds") if 'Item' in table.get_item(Key={'p': p1}): print("p1 alive") time.sleep(duration / 15) # After the delay, p2 should be alive, p1 should not assert 'Item' in table.get_item(Key={'p': p1})
def test_ttl_expiration_long(dynamodb): # Write 100*N items to the table, 1/100th of them are expired. The test # completes when all of them are expired (or on timeout). # To compilicate matter for the paging that we're trying to test, # have N partitions with 100 items in each, so potentially the paging # might stop in the middle of a partition. # We need to pick a relatively big N to cause the 100*N items to exceed # the size of a single page (I tested this by stopping the scanner after # the first page and checking which N starts to generate incorrect results) N = 400 max_duration = 1200 if is_aws(dynamodb) else 15 with new_test_table(dynamodb, KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }], AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'N' }, { 'AttributeName': 'c', 'AttributeType': 'N' }]) as table: ttl_spec = {'AttributeName': 'expiration', 'Enabled': True} response = table.meta.client.update_time_to_live( TableName=table.name, TimeToLiveSpecification=ttl_spec) with table.batch_writer() as batch: for p in range(N): for c in range(100): # Only the first item (c==0) in each partition will expire expiration = int(time.time()) - 60 if c == 0 else int( time.time()) + 3600 batch.put_item(Item={ 'p': p, 'c': c, 'expiration': expiration }) start_time = time.time() while time.time() < start_time + max_duration: # We could have used Select=COUNT here, but Alternator doesn't # implement it yet (issue #5058). count = 0 response = table.scan(ConsistentRead=True) if 'Count' in response: count += response['Count'] while 'LastEvaluatedKey' in response: response = table.scan( ExclusiveStartKey=response['LastEvaluatedKey'], ConsistentRead=True) if 'Count' in response: count += response['Count'] print(count) if count == 99 * N: break time.sleep(max_duration / 15.0) assert count == 99 * N
def test_ttl_expiration_with_rangekey(dynamodb): # Note that unlike test_ttl_expiration, this test doesn't have a fixed # duration - it finishes as soon as the item we expect to be expired # is expired. max_duration = 1200 if is_aws(dynamodb) else 10 with new_test_table(dynamodb, KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }], AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'S' }]) as table: # Enable TTL, using the attribute "expiration": client = table.meta.client ttl_spec = {'AttributeName': 'expiration', 'Enabled': True} response = client.update_time_to_live(TableName=table.name, TimeToLiveSpecification=ttl_spec) assert response['TimeToLiveSpecification'] == ttl_spec # This item should never expire, it is missing the "expiration" # attribute: p1 = random_string() c1 = random_string() table.put_item(Item={'p': p1, 'c': c1}) # This item should expire ASAP, as its "expiration" has already # passed, one minute ago: p2 = random_string() c2 = random_string() table.put_item(Item={ 'p': p2, 'c': c2, 'expiration': int(time.time()) - 60 }) start_time = time.time() while time.time() < start_time + max_duration: if ('Item' in table.get_item(Key={ 'p': p1, 'c': c1 })) and not ('Item' in table.get_item(Key={ 'p': p2, 'c': c2 })): # p2 expired, p1 didn't. We're done break time.sleep(max_duration / 15.0) assert 'Item' in table.get_item(Key={'p': p1, 'c': c1}) assert not 'Item' in table.get_item(Key={'p': p2, 'c': c2})
def check_pre_raft(dynamodb): from util import is_aws # If not running on Scylla, return false. if is_aws(dynamodb): return false # In Scylla, we check Raft mode by inspecting the configuration via a # system table (which is also visible in Alternator) config_table = dynamodb.Table('.scylla.alternator.system.config') experimental_features = config_table.query( KeyConditionExpression='#key=:val', ExpressionAttributeNames={'#key': 'name'}, ExpressionAttributeValues={':val': 'experimental_features' })['Items'][0]['value'] return not '"raft"' in experimental_features
def test_ttl_expiration_range(dynamodb): duration = 1200 if is_aws(dynamodb) else 10 with new_test_table(dynamodb, KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }], AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'N' }]) as table: ttl_spec = {'AttributeName': 'c', 'Enabled': True} table.meta.client.update_time_to_live(TableName=table.name, TimeToLiveSpecification=ttl_spec) # c1 is in the past, and should be expired. c2 is in the distant # future and should not be expired. p = random_string() c1 = int(time.time()) - 60 c2 = int(time.time()) + 3600 table.put_item(Item={'p': p, 'c': c1}) table.put_item(Item={'p': p, 'c': c2}) start_time = time.time() def check_expired(): return not 'Item' in table.get_item(Key={ 'p': p, 'c': c1 }) and 'Item' in table.get_item(Key={ 'p': p, 'c': c2 }) while time.time() < start_time + duration: print(f"--- {int(time.time()-start_time)} seconds") if 'Item' in table.get_item(Key={'p': p, 'c': c1}): print("c1 alive") if 'Item' in table.get_item(Key={'p': p, 'c': c2}): print("c2 alive") if check_expired(): break time.sleep(duration / 15) # After the delay, c2 should be alive, c1 should not assert check_expired()
def rest_api(dynamodb): if is_aws(dynamodb): pytest.skip('Scylla-only REST API not supported by AWS') url = dynamodb.meta.client._endpoint.host # The REST API is on port 10000, and always http, not https. url = re.sub(r':[0-9]+(/|$)', ':10000', url) url = re.sub(r'^https:', 'http:', url) # Scylla's REST API does not have an official "ping" command, # so we just list the keyspaces as a (usually) short operation try: requests.get(f'{url}/column_family/name/keyspace', timeout=1).raise_for_status() except: pytest.skip('Cannot connect to Scylla REST API') return url
def scylla_only(dynamodb): if is_aws(dynamodb): pytest.skip('Scylla-only feature not supported by AWS')
def test_ttl_expiration_streams(dynamodb, dynamodbstreams): # In my experiments, a 30-minute (1800 seconds) is the typical # expiration delay in this test. If the test doesn't finish within # max_duration, we report a failure. max_duration = 3600 if is_aws(dynamodb) else 10 with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }, ], AttributeDefinitions=[ { 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'S' }, ], StreamSpecification={ 'StreamEnabled': True, 'StreamViewType': 'NEW_AND_OLD_IMAGES' }) as table: ttl_spec = {'AttributeName': 'expiration', 'Enabled': True} table.meta.client.update_time_to_live(TableName=table.name, TimeToLiveSpecification=ttl_spec) # Before writing to the table, wait for the stream to become active # so we can be sure that the expiration - even if it miraculously # happens in a second (it usually takes 30 minutes) - is guaranteed # to reach the stream. stream_enabled = False start_time = time.time() while time.time() < start_time + 120: desc = table.meta.client.describe_table( TableName=table.name)['Table'] if 'LatestStreamArn' in desc: arn = desc['LatestStreamArn'] desc = dynamodbstreams.describe_stream(StreamArn=arn) if desc['StreamDescription']['StreamStatus'] == 'ENABLED': stream_enabled = True break time.sleep(10) assert stream_enabled # Write a single expiring item. Set its expiration one minute in the # past, so the item should expire ASAP. p = random_string() c = random_string() expiration = int(time.time()) - 60 table.put_item(Item={ 'p': p, 'c': c, 'animal': 'dog', 'expiration': expiration }) # Wait (up to max_duration) for the item to expire, and for the # expiration event to reach the stream: start_time = time.time() event_found = False while time.time() < start_time + max_duration: print(f"--- {int(time.time()-start_time)} seconds") item_expired = not 'Item' in table.get_item(Key={'p': p, 'c': c}) for record in read_entire_stream(dynamodbstreams, table): # An expired item has a specific userIdentity as follows: if record.get('userIdentity') == { 'Type': 'Service', 'PrincipalId': 'dynamodb.amazonaws.com' }: # The expired item should be a REMOVE, and because we # asked for old images and both the key and the full # content. assert record['eventName'] == 'REMOVE' assert record['dynamodb']['Keys'] == { 'p': { 'S': p }, 'c': { 'S': c } } assert record['dynamodb']['OldImage'] == { 'p': { 'S': p }, 'c': { 'S': c }, 'animal': { 'S': 'dog' }, 'expiration': { 'N': str(expiration) } } event_found = True print(f"item expired {item_expired} event {event_found}") if item_expired and event_found: return time.sleep(max_duration / 15) pytest.fail('item did not expire or event did not reach stream')
def test_ttl_expiration_lsi_key(dynamodb): # In my experiments, a 30-minute (1800 seconds) is the typical delay # for expiration in this test for DynamoDB max_duration = 3600 if is_aws(dynamodb) else 10 with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }, ], LocalSecondaryIndexes=[ { 'IndexName': 'lsi', 'KeySchema': [ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'l', 'KeyType': 'RANGE' }, ], 'Projection': { 'ProjectionType': 'ALL' }, }, ], AttributeDefinitions=[ { 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'S' }, { 'AttributeName': 'l', 'AttributeType': 'N' }, ]) as table: # The expiration-time attribute is the LSI key "l": ttl_spec = {'AttributeName': 'l', 'Enabled': True} response = table.meta.client.update_time_to_live( TableName=table.name, TimeToLiveSpecification=ttl_spec) assert 'TimeToLiveSpecification' in response assert response['TimeToLiveSpecification'] == ttl_spec p = random_string() c = random_string() l = random_string() # expiration one minute in the past, so item should expire ASAP. expiration = int(time.time()) - 60 table.put_item(Item={'p': p, 'c': c, 'l': expiration}) start_time = time.time() gsi_was_alive = False while time.time() < start_time + max_duration: print(f"--- {int(time.time()-start_time)} seconds") if 'Item' in table.get_item(Key={'p': p, 'c': c}): print("base alive") else: # test is done - and successful: return time.sleep(max_duration / 15) pytest.fail('base not expired')
def test_ttl_expiration_gsi_lsi(dynamodb): # In our experiments we noticed that when a table has secondary indexes, # items are expired with significant delay. Whereas a 10 minute delay # for regular tables was typical, in the table we have here we saw # a typical delay of 30 minutes before items expired. max_duration = 3600 if is_aws(dynamodb) else 10 with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' }, ], LocalSecondaryIndexes=[ { 'IndexName': 'lsi', 'KeySchema': [ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'l', 'KeyType': 'RANGE' }, ], 'Projection': { 'ProjectionType': 'ALL' }, }, ], GlobalSecondaryIndexes=[ { 'IndexName': 'gsi', 'KeySchema': [ { 'AttributeName': 'g', 'KeyType': 'HASH' }, ], 'Projection': { 'ProjectionType': 'ALL' } }, ], AttributeDefinitions=[ { 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'S' }, { 'AttributeName': 'g', 'AttributeType': 'S' }, { 'AttributeName': 'l', 'AttributeType': 'S' }, ]) as table: ttl_spec = {'AttributeName': 'expiration', 'Enabled': True} response = table.meta.client.update_time_to_live( TableName=table.name, TimeToLiveSpecification=ttl_spec) assert 'TimeToLiveSpecification' in response assert response['TimeToLiveSpecification'] == ttl_spec p = random_string() c = random_string() g = random_string() l = random_string() # expiration one minute in the past, so item should expire ASAP. expiration = int(time.time()) - 60 table.put_item(Item={ 'p': p, 'c': c, 'g': g, 'l': l, 'expiration': expiration }) start_time = time.time() gsi_was_alive = False while time.time() < start_time + max_duration: print(f"--- {int(time.time()-start_time)} seconds") base_alive = 'Item' in table.get_item(Key={'p': p, 'c': c}) gsi_alive = bool( full_query(table, IndexName='gsi', ConsistentRead=False, KeyConditionExpression="g=:g", ExpressionAttributeValues={':g': g})) lsi_alive = bool( full_query(table, IndexName='lsi', KeyConditionExpression="p=:p and l=:l", ExpressionAttributeValues={ ':p': p, ':l': l })) if base_alive: print("base alive") if gsi_alive: print("gsi alive") # gsi takes time to go up, so make sure it did gsi_was_alive = True if lsi_alive: print("lsi alive") # If the base item, gsi item and lsi item have all expired, the # test is done - and successful: if not base_alive and not gsi_alive and gsi_was_alive and not lsi_alive: return time.sleep(max_duration / 15) pytest.fail('base, gsi, or lsi not expired')
def test_ttl_expiration(dynamodb): duration = 1200 if is_aws(dynamodb) else 10 # delta is a quarter of the test duration, but no less than one second, # and we use it to schedule some expirations a bit after the test starts, # not immediately. delta = math.ceil(duration / 4) assert delta >= 1 with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, ], AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }]) as table: # Insert one expiring item *before* enabling the TTL, to verify that # items that already exist when TTL is configured also get handled. p0 = random_string() table.put_item(Item={'p': p0, 'expiration': int(time.time()) - 60}) # Enable TTL, using the attribute "expiration": client = table.meta.client ttl_spec = {'AttributeName': 'expiration', 'Enabled': True} response = client.update_time_to_live(TableName=table.name, TimeToLiveSpecification=ttl_spec) assert response['TimeToLiveSpecification'] == ttl_spec # This item should never expire, it is missing the "expiration" # attribute: p1 = random_string() table.put_item(Item={'p': p1}) # This item should expire ASAP, as its "expiration" has already # passed, one minute ago: p2 = random_string() table.put_item(Item={'p': p2, 'expiration': int(time.time()) - 60}) # This item has an expiration more than 5 years in the past (it is my # birth date...), so according to the DynamoDB documentation it should # be ignored and p3 should never expire: p3 = random_string() table.put_item(Item={'p': p3, 'expiration': 162777600}) # This item has as its expiration delta into the future, which is a # small part of the test duration, so should expire by the time the # test ends: p4 = random_string() table.put_item(Item={'p': p4, 'expiration': int(time.time()) + delta}) # This item starts with expiration time delta into the future, # but before it expires we will move it again, so it will never expire: p5 = random_string() table.put_item(Item={'p': p5, 'expiration': int(time.time()) + delta}) # This item has an expiration time two durations into the future, so it # will not expire by the time the test ends: p6 = random_string() table.put_item(Item={ 'p': p6, 'expiration': int(time.time() + duration * 2) }) # Like p4, this item has an expiration time delta into the future, # here the expiration time is wrongly encoded as a string, not a # number, so the item should never expire: p7 = random_string() table.put_item(Item={ 'p': p7, 'expiration': str(int(time.time()) + delta) }) # Like p2, p8 and p9 also have an already-passed expiration time, # and should expire ASAP. However, whereas p2 had a straighforward # integer like 12345678 as the expiration time, p8 and p9 have # slightly more elaborate numbers: p8 has 1234567e1 and p9 has # 12345678.1234. Those formats should be fine, and this test verifies # the TTL code's number parsing doesn't get confused (in our original # implementation, it did). p8 = random_string() with client_no_transform(table.meta.client) as client: client.put_item(TableName=table.name, Item={ 'p': { 'S': p8 }, 'expiration': { 'N': str( (int(time.time()) - 60) // 10) + "e1" } }) # Similarly, floating point expiration time like 12345678.1 should # also be fine (note that Python's time.time() returns floating point). # This item should also be expired ASAP too. p9 = random_string() print(Decimal(str(time.time() - 60))) table.put_item(Item={ 'p': p9, 'expiration': Decimal(str(time.time() - 60)) }) def check_expired(): # After the delay, p1,p3,p5,p6,p7 should be alive, p0,p2,p4 should not return not 'Item' in table.get_item(Key={'p': p0}) \ and 'Item' in table.get_item(Key={'p': p1}) \ and not 'Item' in table.get_item(Key={'p': p2}) \ and 'Item' in table.get_item(Key={'p': p3}) \ and not 'Item' in table.get_item(Key={'p': p4}) \ and 'Item' in table.get_item(Key={'p': p5}) \ and 'Item' in table.get_item(Key={'p': p6}) \ and 'Item' in table.get_item(Key={'p': p7}) \ and not 'Item' in table.get_item(Key={'p': p8}) \ and not 'Item' in table.get_item(Key={'p': p9}) # We could have just done time.sleep(duration) here, but in case a # user is watching this long test, let's output the status every # minute, and it also allows us to test what happens when an item # p5's expiration time is continuously pushed back into the future: start_time = time.time() while time.time() < start_time + duration: print(f"--- {int(time.time()-start_time)} seconds") if 'Item' in table.get_item(Key={'p': p0}): print("p0 alive") if 'Item' in table.get_item(Key={'p': p1}): print("p1 alive") if 'Item' in table.get_item(Key={'p': p2}): print("p2 alive") if 'Item' in table.get_item(Key={'p': p3}): print("p3 alive") if 'Item' in table.get_item(Key={'p': p4}): print("p4 alive") if 'Item' in table.get_item(Key={'p': p5}): print("p5 alive") if 'Item' in table.get_item(Key={'p': p6}): print("p6 alive") if 'Item' in table.get_item(Key={'p': p7}): print("p7 alive") if 'Item' in table.get_item(Key={'p': p8}): print("p8 alive") if 'Item' in table.get_item(Key={'p': p9}): print("p9 alive") # Always keep p5's expiration delta into the future table.update_item(Key={'p': p5}, AttributeUpdates={ 'expiration': { 'Value': int(time.time()) + delta, 'Action': 'PUT' } }) if check_expired(): break time.sleep(duration / 15.0) assert check_expired()