def test_fuzzing_with_proxy_multi_and_restarts(cluster, workload):
"""
Test proxy with transaction safety and random node restarts.
"""
nodes = 3
cycles = 20
thread_count = 200
cluster.create(nodes,
raft_args={
'raftize-all-commands': 'yes',
'follower-proxy': 'yes'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
for i in range(cycles):
time.sleep(1)
try:
logging.info('Cycle %s: %s', i, workload.stats())
cluster.random_node().restart()
except ResponseError as err:
logging.error('Remove node: %s', err)
continue
logging.info('All cycles finished')
workload.stop()
def test_log_fixup_after_snapshot_delivery(cluster):
"""
Log must be restarted when loading a snapshot.
"""
cluster.create(3)
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(2).terminate()
cluster.node(1).raft_exec('INCR', 'key')
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
# Confirm node 2's log starts from 1
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 0
cluster.node(2).start()
cluster.node(2).wait_for_info_param('state', 'up')
# node 2 must get a snapshot to sync, make sure this happens and that
# the log is ok.
cluster.node(2).wait_for_current_index(8)
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 8
def test_full_cluster_remove(cluster):
"""
Remove all cluster nodes.
"""
cluster.create(5)
leader = cluster.leader_node()
expected_nodes = 5
for node_id in (2, 3, 4, 5):
leader.client.execute_command('RAFT.NODE', 'REMOVE', str(node_id))
expected_nodes -= 1
leader.wait_for_num_nodes(expected_nodes)
# make sure other nodes are down
for node_id in (2, 3, 4, 5):
assert not cluster.node(node_id).process_is_up()
# and make sure they start up in uninitialized state
for node_id in (2, 3, 4, 5):
cluster.node(node_id).terminate()
cluster.node(node_id).start()
for node_id in (2, 3, 4, 5):
assert cluster.node(node_id).raft_info()['state'] == 'uninitialized'
def test_fuzzing_with_restarts_and_rewrites(cluster):
"""
Counter fuzzer with log rewrites.
"""
nodes = 3
cycles = 100
cluster.create(nodes)
# Randomize max log entries
for node in cluster.nodes.values():
node.client.execute_command('RAFT.CONFIG', 'SET',
'raft-log-max-file-size',
str(random.randint(1000, 2000)))
for i in range(cycles):
assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
logging.info('---------- Executed INCRBY # %s', i)
if random.randint(1, 7) == 1:
r = random.randint(1, nodes)
logging.info('********** Restarting node %s **********', r)
cluster.node(r).restart()
cluster.node(r).wait_for_election()
logging.info('********** Node %s is UP **********', r)
assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_rolled_back_read_only_multi_reply(cluster):
"""
Watch the reply to a MULTI operation that ends up being discarded
due to election change before commit.
"""
cluster.create(3)
cluster.node(1).raft_exec('SET', 'key', 'value')
cluster.node(2).pause()
cluster.node(3).pause()
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'MULTI')
assert conn.read_response() == b'OK'
conn.send_command('RAFT', 'GET', 'key')
assert conn.read_response() == b'QUEUED'
conn.send_command('RAFT', 'EXEC')
cluster.node(1).pause()
cluster.node(2).kill()
cluster.node(3).kill()
cluster.node(2).start()
cluster.node(3).start()
cluster.node(2).wait_for_election()
cluster.node(1).resume()
with raises(ResponseError, match='TIMEOUT'):
assert conn.read_response() == None
def test_proxying(cluster):
"""
Command proxying from follower to leader works
"""
cluster.create(3)
assert cluster.leader == 1
with raises(ResponseError, match='MOVED'):
assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK'
assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET',
'follower-proxy',
'yes') == b'OK'
# Basic sanity
assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK'
assert cluster.raft_exec('GET', 'key') == b'value'
# Numeric values
assert cluster.node(2).raft_exec('SADD', 'myset', 'a') == 1
assert cluster.node(2).raft_exec('SADD', 'myset', 'b') == 1
# Multibulk
assert set(cluster.node(2).raft_exec('SMEMBERS',
'myset')) == set([b'a', b'b'])
# Nested multibulk
assert set(
cluster.node(2).raft_exec('EVAL', 'return {{\'a\',\'b\',\'c\'}};',
0)[0]) == set([b'a', b'b', b'c'])
# Error
with raises(ResponseError, match='WRONGTYPE'):
cluster.node(2).raft_exec('INCR', 'myset')
def test_stability_with_snapshots_and_restarts(cluster, workload):
"""
Test stability of the cluster with frequent snapshoting.
"""
thread_count = 100
duration = 300
cluster.create(5,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes',
'raft-log-max-file-size': '2000'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
# Monitor progress
start = time.time()
last_commit_index = 0
while start + duration > time.time():
time.sleep(2)
cluster.random_node().restart()
workload.stop()
def test_readonly_commands(cluster):
"""
Test read-only command execution, which does not go through the Raft
log.
"""
cluster.create(3)
assert cluster.leader == 1
# Write something
assert cluster.node(1).current_index() == 5
assert cluster.node(1).raft_exec('SET', 'key', 'value') == b'OK'
assert cluster.node(1).current_index() == 6
# Read something, log should not grow
assert cluster.node(1).raft_exec('GET', 'key') == b'value'
assert cluster.node(1).current_index() == 6
# Tear down cluster, reads should hang
cluster.node(2).terminate()
cluster.node(3).terminate()
conn = cluster.node(1).client.connection_pool.get_connection(
'RAFT', socket_timeout=1)
conn.send_command('RAFT', 'GET', 'key')
assert not conn.can_read(timeout=1)
# Now configure non-quorum reads
cluster.node(1).raft_config_set('quorum-reads', 'no')
assert cluster.node(1).raft_exec('GET', 'key') == b'value'
def test_proxy_stability_under_load(cluster, workload):
"""
Test stability of the cluster with follower proxy under load.
"""
thread_count = 500
duration = 300
cluster.create(5,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
# Monitor progress
start = time.time()
last_commit_index = 0
while start + duration > time.time():
time.sleep(2)
new_commit_index = cluster.node(cluster.leader).commit_index()
assert new_commit_index > last_commit_index
last_commit_index = new_commit_index
workload.stop()
def test_uncommitted_log_rewrite(cluster):
cluster.create(3)
# Log contains 5 config entries
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', 'value') # Entry idx 6
cluster.node(2).terminate()
cluster.node(3).terminate()
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'SET', 'key2', 'value2') # Entry idx 7
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 1
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 1
cluster.node(1).kill()
cluster.node(1).start()
cluster.node(1).wait_for_info_param('state', 'up')
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).raft_info()['log_entries'] == 1
def test_cfg_node_removed_from_snapshot(cluster):
"""
Node able to learn that another node left by reading the snapshot metadata.
"""
cluster.create(5)
cluster.node(1).raft_exec('SET', 'key', 'value')
cluster.wait_for_unanimity()
# interrupt
# we now take down node 4 so it doesn't get updates and remove node 5.
cluster.node(4).terminate()
cluster.remove_node(5)
cluster.wait_for_unanimity(exclude=[4])
cluster.node(1).wait_for_log_applied()
assert cluster.node(1).raft_info()['num_nodes'] == 4
# now compact logs
cluster.wait_for_unanimity(exclude=[4])
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 0
# bring back node 4
cluster.node(4).start()
cluster.node(4).wait_for_election()
cluster.wait_for_unanimity()
assert cluster.node(4).raft_info()['num_nodes'] == 4
def test_config_from_second_generation_snapshot(cluster):
"""
A regression test for #44: confirm that if we load a snapshot
on startup, do nothing, then re-create a snapshot we don't end
up with a messed up nodes config.
"""
cluster.create(3)
# Bump the log a bit
for _ in range(20):
assert cluster.raft_exec('INCR', 'testkey')
# Compact to get rid of logs
node3 = cluster.node(3)
assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'
# Restart node
node3.restart()
node3.wait_for_node_voting()
# Bump the log a bit
for _ in range(20):
assert cluster.raft_exec('INCR', 'testkey')
# Recompact
cluster.wait_for_unanimity()
assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'
node3.restart()
node3.wait_for_node_voting()
assert node3.raft_info()['num_nodes'] == 3
def test_removed_node_remains_dead(cluster):
"""
A removed node stays down and does not resurrect in any case.
"""
cluster.create(3)
# Some baseline data
for _ in range(100):
cluster.raft_exec('INCR', 'counter')
# Remove node 3
cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '3')
cluster.node(1).wait_for_num_voting_nodes(2)
# Add more data
for _ in range(100):
cluster.raft_exec('INCR', 'counter')
# Check
node = cluster.node(3)
# Verify node 3 does not accept writes
with raises(RedisError):
node.client.execute_command('RAFT', 'INCR', 'counter')
# Verify node 3 still does not accept writes after a restart
node.terminate()
node.start()
with raises(RedisError):
node.client.execute_command('RAFT', 'INCR', 'counter')
def test_snapshot_delivery(cluster):
"""
Ability to properly deliver and load a snapshot.
"""
cluster.create(3, raft_args={'raftize-all-commands': 'yes'})
n1 = cluster.node(1)
n1.raft_exec('INCR', 'testkey')
n1.raft_exec('INCR', 'testkey')
n1.raft_exec('INCR', 'testkey')
for i in range(1000):
pipe = n1.client.pipeline(transaction=True)
for j in range(100):
pipe.rpush('list-%s' % i, 'elem-%s' % j)
pipe.execute()
cluster.node(3).terminate()
n1.raft_exec('SETRANGE', 'bigkey', '104857600', 'x')
n1.raft_exec('INCR', 'testkey')
assert n1.client.get('testkey') == b'4'
assert n1.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'
assert n1.raft_info()['log_entries'] == 0
n3 = cluster.node(3)
n3.start()
n1.raft_exec('INCR', 'testkey')
n3.wait_for_node_voting()
cluster.wait_for_unanimity()
n3.wait_for_log_applied()
n3.client.execute_command('RAFT.CONFIG', 'SET',
'raftize-all-commands', 'no')
assert n3.client.get('testkey') == b'5'
def test_rolled_back_reply(cluster):
"""
Watch the reply to a write operation that ends up being discarded
due to election change before commit.
"""
cluster.create(3)
cluster.node(2).pause()
cluster.node(3).pause()
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'INCR', 'key')
cluster.node(1).pause()
cluster.node(2).kill()
cluster.node(3).kill()
cluster.node(2).start()
cluster.node(3).start()
cluster.node(2).wait_for_election()
cluster.node(1).resume()
with raises(ResponseError, match='TIMEOUT'):
conn.read_response()
def test_leader_removal_not_allowed(cluster):
"""
Leader node cannot be removed.
"""
cluster.create(3)
assert cluster.leader == 1
with raises(ResponseError, match='cannot remove leader'):
cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '1')
def test_update_self_voting_state_from_snapshot(cluster):
cluster.create(3)
assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'NODECFG', '2', '-voting') == b'OK'
assert cluster.node(2).raft_info()['is_voting'] == 'yes'
assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'
cluster.node(1).client.execute_command('RAFT.DEBUG', 'SENDSNAPSHOT', '2')
cluster.node(2).wait_for_info_param('snapshots_loaded', 1)
assert cluster.node(2).raft_info()['is_voting'] == 'no'
def test_reelection_basic_flow(cluster):
"""
Basic reelection flow
"""
cluster.create(3)
assert cluster.leader == 1
assert cluster.raft_exec('SET', 'key', 'value') == b'OK'
cluster.node(1).terminate()
cluster.node(2).wait_for_election()
assert cluster.raft_exec('SET', 'key2', 'value2') == b'OK'
cluster.exec_all('GET', 'key2')
def test_index_correct_right_after_snapshot(cluster):
cluster.create(1)
for _ in range(10):
cluster.node(1).raft_exec('INCR', 'counter')
info = cluster.node(1).raft_info()
assert info['current_index'] == 11
# Make sure log is compacted
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
info = cluster.node(1).raft_info()
assert info['log_entries'] == 0
assert info['current_index'] == 11
assert info['commit_index'] == 11
def test_reply_to_cache_invalidated_entry(cluster):
"""
Reply a RAFT redis command that have its entry already removed
from the cache.
"""
cluster.create(3)
assert cluster.leader == 1
# Configure a small cache
assert cluster.node(1).raft_config_set('raft-log-max-cache-size', '1kb')
# Break cluster to avoid commits
cluster.node(2).terminate()
cluster.node(3).terminate()
# Send commands that are guarnateed to overflow the cache
conns = []
for i in range(10):
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'SET', 'key%s' % i, 'x' * 1000)
conns.append(conn)
# give periodic job time to handle cache
time.sleep(0.5)
# confirm all raft entries were created but some have been evicted
# from cache already.
info = cluster.node(1).raft_info()
assert info['log_entries'] == 15
assert info['cache_entries'] < 10
# Repair cluster and wait
cluster.node(2).start()
cluster.node(3).start()
cluster.node(1).wait_for_num_voting_nodes(3)
time.sleep(1)
assert cluster.node(1).commit_index() == 15
# Expect TIMEOUT or OK for all
for conn in conns:
assert conn.can_read(timeout=1)
try:
assert conn.read_response() == b'OK'
except ResponseError as err:
assert str(err).startswith('TIMEOUT')
def test_stale_reads_on_leader_election(cluster):
"""
"""
cluster.create(3)
# Try 10 times
for _ in range(10):
val_written = cluster.raft_exec("INCR", "counter-1")
leader = cluster.node(cluster.leader)
leader.terminate()
leader.start(verify=False)
val_read = cluster.raft_exec('GET', 'counter-1')
assert val_read is not None
assert val_written == int(val_read)
time.sleep(1)
def test_log_rollback(cluster):
"""
Rollback of log entries that were written in the minority.
"""
cluster.create(3)
assert cluster.leader == 1
assert cluster.raft_exec('INCRBY', 'key', '111') == 111
# Break cluster
cluster.node(2).terminate()
cluster.node(3).terminate()
# Load a command which can't be committed
assert cluster.node(1).current_index() == 6
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'INCRBY', 'key', '222')
assert cluster.node(1).current_index() == 7
cluster.node(1).terminate()
# We want to be sure the last entry is in the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count() == 7
# Restart the cluster without node 1, make sure the write was
# not committed.
cluster.node(2).start()
cluster.node(3).start()
cluster.node(2).wait_for_election()
assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry
# Restart node 1
cluster.node(1).start()
cluster.node(1).wait_for_election()
# Make another write and make sure it overwrites the previous one in
# node 1's log
assert cluster.raft_exec('INCRBY', 'key', '333') == 444
cluster.wait_for_unanimity()
# Make sure log reflects the change
log.reset()
log.read()
assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
def test_new_uncommitted_during_rewrite(cluster):
cluster.create(3)
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', '1')
# Initiate compaction and wait to see it's in progress
conn = cluster.node(1).client.connection_pool.get_connection('COMPACT')
conn.send_command('RAFT.DEBUG', 'COMPACT', '2')
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes')
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
# Send a bunch of writes
cluster.node(1).raft_exec('INCRBY', 'key', '2')
cluster.node(1).raft_exec('INCRBY', 'key', '3')
cluster.node(1).raft_exec('INCRBY', 'key', '4')
# Wait for compaction to end
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no')
# Make sure our writes made it to the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 3
# Extra check -- Make sure we can read it back. Note that we need to start
# all nodes because we don't log the commit index.
cluster.node(1).terminate()
cluster.node(2).terminate()
cluster.node(3).terminate()
cluster.node(1).start()
cluster.node(2).start()
cluster.node(3).start()
cluster.node(1).wait_for_info_param('state', 'up')
# Make sure cluster state is as expected
assert cluster.raft_exec('get', 'key') == b'10'
# Make sure node 1 state is as expected
cluster.node(1).wait_for_log_applied()
assert cluster.node(1).client.get('key') == b'10'
def test_all_committed_log_rewrite(cluster):
"""
Log rewrite operation when all entries are committed, so we expect an
empty log.
"""
cluster.create(3)
cluster.node(1).raft_exec('SET', 'key1', 'value')
cluster.node(1).raft_exec('SET', 'key2', 'value')
cluster.node(1).raft_exec('SET', 'key3', 'value')
cluster.wait_for_unanimity()
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 0
# Make sure we have no log entries!
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 0
def test_multi_exec_proxying(cluster):
"""
Proxy a MULTI/EXEC sequence
"""
cluster.create(3)
assert cluster.leader == 1
assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET',
'follower-proxy',
'yes') == b'OK'
# Basic sanity
n2 = cluster.node(2)
assert n2.raft_info()['current_index'] == 5
assert n2.raft_exec('MULTI')
assert n2.raft_exec('INCR', 'key') == b'QUEUED'
assert n2.raft_exec('INCR', 'key') == b'QUEUED'
assert n2.raft_exec('INCR', 'key') == b'QUEUED'
assert n2.raft_exec('EXEC') == [1, 2, 3]
assert n2.raft_info()['current_index'] == 6
def test_stale_reads_on_restarts(cluster, workload):
"""
Test proxy mode with MULTI transactions safety checks and
reconnections (dropping clients with CLIENT KILL).
"""
thread_count = 50
cycles = 20
cluster.create(3,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes'
})
workload.start(thread_count, cluster, MonotonicIncrCheck)
for _ in range(cycles):
time.sleep(1)
cluster.restart()
logging.info('All cycles finished')
workload.stop()
def test_node_history_with_same_address(cluster):
""
""
cluster.create(5)
cluster.raft_exec("INCR", "step-counter")
# Remove nodes
ports = []
for node_id in [2, 3, 4, 5]:
ports.append(cluster.node(node_id).port)
cluster.remove_node(node_id)
cluster.leader_node().wait_for_log_applied()
cluster.node(cluster.leader).wait_for_num_nodes(1)
# Now add and remove several more times
for _ in range(5):
for port in ports:
n = cluster.add_node(port=port)
cluster.leader_node().wait_for_num_nodes(2)
cluster.leader_node().wait_for_log_applied()
cluster.remove_node(n.id)
cluster.leader_node().wait_for_num_nodes(1)
cluster.leader_node().wait_for_log_applied()
# Add enough data in the log to satisfy timing
for _ in range(3000):
cluster.raft_exec("INCR", "step-counter")
# Add another node
new_node = cluster.add_node(port=ports[0])
new_node.wait_for_node_voting()
# Terminate all
cluster.terminate()
# Start new node
cluster.start()
# need some time to start applying logs..
time.sleep(2)
assert cluster.raft_exec("GET", "step-counter") == b'3001'
def test_fuzzing_with_restarts(cluster):
"""
Basic Raft fuzzer test
"""
nodes = 3
cycles = 100
cluster.create(nodes)
for i in range(cycles):
assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
logging.info('---------- Executed INCRBY # %s', i)
if i % 7 == 0:
r = random.randint(1, nodes)
logging.info('********** Restarting node %s **********', r)
cluster.node(r).restart()
cluster.node(r).wait_for_election()
logging.info('********** Node %s is UP **********', r)
assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_fuzzing_with_config_changes(cluster):
"""
Basic Raft fuzzer test
"""
nodes = 5
cycles = 100
cluster.create(nodes)
for i in range(cycles):
assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
if random.randint(1, 7) == 1:
try:
node_id = cluster.random_node_id()
cluster.remove_node(node_id)
except ResponseError:
continue
cluster.add_node().wait_for_node_voting()
assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_single_voting_change_enforced(cluster):
"""
A single concurrent voting change is enforced when removing nodes.
"""
cluster.create(5)
assert cluster.leader == 1
# Simulate a partition
cluster.node(2).terminate()
cluster.node(3).terminate()
cluster.node(4).terminate()
assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE',
'5') == b'OK'
with raises(ResponseError, match='a voting change is already in progress'):
assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE',
'4') == b'OK'
time.sleep(1)
assert cluster.node(1).raft_info()['num_nodes'] == 5