def test_fuzzing_with_proxy_multi_and_restarts(cluster, workload): """ Test proxy with transaction safety and random node restarts. """ nodes = 3 cycles = 20 thread_count = 200 cluster.create(nodes, raft_args={ 'raftize-all-commands': 'yes', 'follower-proxy': 'yes' }) workload.start(thread_count, cluster, MultiWithLargeReply) for i in range(cycles): time.sleep(1) try: logging.info('Cycle %s: %s', i, workload.stats()) cluster.random_node().restart() except ResponseError as err: logging.error('Remove node: %s', err) continue logging.info('All cycles finished') workload.stop()
def test_log_fixup_after_snapshot_delivery(cluster): """ Log must be restarted when loading a snapshot. """ cluster.create(3) cluster.node(1).raft_exec('INCR', 'key') cluster.node(1).raft_exec('INCR', 'key') cluster.node(2).terminate() cluster.node(1).raft_exec('INCR', 'key') assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' # Confirm node 2's log starts from 1 log = RaftLog(cluster.node(2).raftlog) log.read() assert log.header().snapshot_index() == 0 cluster.node(2).start() cluster.node(2).wait_for_info_param('state', 'up') # node 2 must get a snapshot to sync, make sure this happens and that # the log is ok. cluster.node(2).wait_for_current_index(8) log = RaftLog(cluster.node(2).raftlog) log.read() assert log.header().snapshot_index() == 8
def test_full_cluster_remove(cluster): """ Remove all cluster nodes. """ cluster.create(5) leader = cluster.leader_node() expected_nodes = 5 for node_id in (2, 3, 4, 5): leader.client.execute_command('RAFT.NODE', 'REMOVE', str(node_id)) expected_nodes -= 1 leader.wait_for_num_nodes(expected_nodes) # make sure other nodes are down for node_id in (2, 3, 4, 5): assert not cluster.node(node_id).process_is_up() # and make sure they start up in uninitialized state for node_id in (2, 3, 4, 5): cluster.node(node_id).terminate() cluster.node(node_id).start() for node_id in (2, 3, 4, 5): assert cluster.node(node_id).raft_info()['state'] == 'uninitialized'
def test_fuzzing_with_restarts_and_rewrites(cluster): """ Counter fuzzer with log rewrites. """ nodes = 3 cycles = 100 cluster.create(nodes) # Randomize max log entries for node in cluster.nodes.values(): node.client.execute_command('RAFT.CONFIG', 'SET', 'raft-log-max-file-size', str(random.randint(1000, 2000))) for i in range(cycles): assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1 logging.info('---------- Executed INCRBY # %s', i) if random.randint(1, 7) == 1: r = random.randint(1, nodes) logging.info('********** Restarting node %s **********', r) cluster.node(r).restart() cluster.node(r).wait_for_election() logging.info('********** Node %s is UP **********', r) assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_rolled_back_read_only_multi_reply(cluster): """ Watch the reply to a MULTI operation that ends up being discarded due to election change before commit. """ cluster.create(3) cluster.node(1).raft_exec('SET', 'key', 'value') cluster.node(2).pause() cluster.node(3).pause() conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'MULTI') assert conn.read_response() == b'OK' conn.send_command('RAFT', 'GET', 'key') assert conn.read_response() == b'QUEUED' conn.send_command('RAFT', 'EXEC') cluster.node(1).pause() cluster.node(2).kill() cluster.node(3).kill() cluster.node(2).start() cluster.node(3).start() cluster.node(2).wait_for_election() cluster.node(1).resume() with raises(ResponseError, match='TIMEOUT'): assert conn.read_response() == None
def test_proxying(cluster): """ Command proxying from follower to leader works """ cluster.create(3) assert cluster.leader == 1 with raises(ResponseError, match='MOVED'): assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK' assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET', 'follower-proxy', 'yes') == b'OK' # Basic sanity assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK' assert cluster.raft_exec('GET', 'key') == b'value' # Numeric values assert cluster.node(2).raft_exec('SADD', 'myset', 'a') == 1 assert cluster.node(2).raft_exec('SADD', 'myset', 'b') == 1 # Multibulk assert set(cluster.node(2).raft_exec('SMEMBERS', 'myset')) == set([b'a', b'b']) # Nested multibulk assert set( cluster.node(2).raft_exec('EVAL', 'return {{\'a\',\'b\',\'c\'}};', 0)[0]) == set([b'a', b'b', b'c']) # Error with raises(ResponseError, match='WRONGTYPE'): cluster.node(2).raft_exec('INCR', 'myset')
def test_stability_with_snapshots_and_restarts(cluster, workload): """ Test stability of the cluster with frequent snapshoting. """ thread_count = 100 duration = 300 cluster.create(5, raft_args={ 'follower-proxy': 'yes', 'raftize-all-commands': 'yes', 'raft-log-max-file-size': '2000' }) workload.start(thread_count, cluster, MultiWithLargeReply) # Monitor progress start = time.time() last_commit_index = 0 while start + duration > time.time(): time.sleep(2) cluster.random_node().restart() workload.stop()
def test_readonly_commands(cluster): """ Test read-only command execution, which does not go through the Raft log. """ cluster.create(3) assert cluster.leader == 1 # Write something assert cluster.node(1).current_index() == 5 assert cluster.node(1).raft_exec('SET', 'key', 'value') == b'OK' assert cluster.node(1).current_index() == 6 # Read something, log should not grow assert cluster.node(1).raft_exec('GET', 'key') == b'value' assert cluster.node(1).current_index() == 6 # Tear down cluster, reads should hang cluster.node(2).terminate() cluster.node(3).terminate() conn = cluster.node(1).client.connection_pool.get_connection( 'RAFT', socket_timeout=1) conn.send_command('RAFT', 'GET', 'key') assert not conn.can_read(timeout=1) # Now configure non-quorum reads cluster.node(1).raft_config_set('quorum-reads', 'no') assert cluster.node(1).raft_exec('GET', 'key') == b'value'
def test_proxy_stability_under_load(cluster, workload): """ Test stability of the cluster with follower proxy under load. """ thread_count = 500 duration = 300 cluster.create(5, raft_args={ 'follower-proxy': 'yes', 'raftize-all-commands': 'yes' }) workload.start(thread_count, cluster, MultiWithLargeReply) # Monitor progress start = time.time() last_commit_index = 0 while start + duration > time.time(): time.sleep(2) new_commit_index = cluster.node(cluster.leader).commit_index() assert new_commit_index > last_commit_index last_commit_index = new_commit_index workload.stop()
def test_uncommitted_log_rewrite(cluster): cluster.create(3) # Log contains 5 config entries # Take down majority to create uncommitted entries and check rewrite cluster.node(1).raft_exec('SET', 'key', 'value') # Entry idx 6 cluster.node(2).terminate() cluster.node(3).terminate() conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'SET', 'key2', 'value2') # Entry idx 7 assert cluster.node(1).current_index() == 7 assert cluster.node(1).commit_index() == 6 assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' assert cluster.node(1).raft_info()['log_entries'] == 1 log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 1 cluster.node(1).kill() cluster.node(1).start() cluster.node(1).wait_for_info_param('state', 'up') assert cluster.node(1).current_index() == 7 assert cluster.node(1).commit_index() == 6 assert cluster.node(1).raft_info()['log_entries'] == 1
def test_cfg_node_removed_from_snapshot(cluster): """ Node able to learn that another node left by reading the snapshot metadata. """ cluster.create(5) cluster.node(1).raft_exec('SET', 'key', 'value') cluster.wait_for_unanimity() # interrupt # we now take down node 4 so it doesn't get updates and remove node 5. cluster.node(4).terminate() cluster.remove_node(5) cluster.wait_for_unanimity(exclude=[4]) cluster.node(1).wait_for_log_applied() assert cluster.node(1).raft_info()['num_nodes'] == 4 # now compact logs cluster.wait_for_unanimity(exclude=[4]) assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' assert cluster.node(1).raft_info()['log_entries'] == 0 # bring back node 4 cluster.node(4).start() cluster.node(4).wait_for_election() cluster.wait_for_unanimity() assert cluster.node(4).raft_info()['num_nodes'] == 4
def test_config_from_second_generation_snapshot(cluster): """ A regression test for #44: confirm that if we load a snapshot on startup, do nothing, then re-create a snapshot we don't end up with a messed up nodes config. """ cluster.create(3) # Bump the log a bit for _ in range(20): assert cluster.raft_exec('INCR', 'testkey') # Compact to get rid of logs node3 = cluster.node(3) assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK' # Restart node node3.restart() node3.wait_for_node_voting() # Bump the log a bit for _ in range(20): assert cluster.raft_exec('INCR', 'testkey') # Recompact cluster.wait_for_unanimity() assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK' node3.restart() node3.wait_for_node_voting() assert node3.raft_info()['num_nodes'] == 3
def test_removed_node_remains_dead(cluster): """ A removed node stays down and does not resurrect in any case. """ cluster.create(3) # Some baseline data for _ in range(100): cluster.raft_exec('INCR', 'counter') # Remove node 3 cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '3') cluster.node(1).wait_for_num_voting_nodes(2) # Add more data for _ in range(100): cluster.raft_exec('INCR', 'counter') # Check node = cluster.node(3) # Verify node 3 does not accept writes with raises(RedisError): node.client.execute_command('RAFT', 'INCR', 'counter') # Verify node 3 still does not accept writes after a restart node.terminate() node.start() with raises(RedisError): node.client.execute_command('RAFT', 'INCR', 'counter')
def test_snapshot_delivery(cluster): """ Ability to properly deliver and load a snapshot. """ cluster.create(3, raft_args={'raftize-all-commands': 'yes'}) n1 = cluster.node(1) n1.raft_exec('INCR', 'testkey') n1.raft_exec('INCR', 'testkey') n1.raft_exec('INCR', 'testkey') for i in range(1000): pipe = n1.client.pipeline(transaction=True) for j in range(100): pipe.rpush('list-%s' % i, 'elem-%s' % j) pipe.execute() cluster.node(3).terminate() n1.raft_exec('SETRANGE', 'bigkey', '104857600', 'x') n1.raft_exec('INCR', 'testkey') assert n1.client.get('testkey') == b'4' assert n1.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK' assert n1.raft_info()['log_entries'] == 0 n3 = cluster.node(3) n3.start() n1.raft_exec('INCR', 'testkey') n3.wait_for_node_voting() cluster.wait_for_unanimity() n3.wait_for_log_applied() n3.client.execute_command('RAFT.CONFIG', 'SET', 'raftize-all-commands', 'no') assert n3.client.get('testkey') == b'5'
def test_rolled_back_reply(cluster): """ Watch the reply to a write operation that ends up being discarded due to election change before commit. """ cluster.create(3) cluster.node(2).pause() cluster.node(3).pause() conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'INCR', 'key') cluster.node(1).pause() cluster.node(2).kill() cluster.node(3).kill() cluster.node(2).start() cluster.node(3).start() cluster.node(2).wait_for_election() cluster.node(1).resume() with raises(ResponseError, match='TIMEOUT'): conn.read_response()
def test_leader_removal_not_allowed(cluster): """ Leader node cannot be removed. """ cluster.create(3) assert cluster.leader == 1 with raises(ResponseError, match='cannot remove leader'): cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '1')
def test_update_self_voting_state_from_snapshot(cluster): cluster.create(3) assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'NODECFG', '2', '-voting') == b'OK' assert cluster.node(2).raft_info()['is_voting'] == 'yes' assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK' cluster.node(1).client.execute_command('RAFT.DEBUG', 'SENDSNAPSHOT', '2') cluster.node(2).wait_for_info_param('snapshots_loaded', 1) assert cluster.node(2).raft_info()['is_voting'] == 'no'
def test_reelection_basic_flow(cluster): """ Basic reelection flow """ cluster.create(3) assert cluster.leader == 1 assert cluster.raft_exec('SET', 'key', 'value') == b'OK' cluster.node(1).terminate() cluster.node(2).wait_for_election() assert cluster.raft_exec('SET', 'key2', 'value2') == b'OK' cluster.exec_all('GET', 'key2')
def test_index_correct_right_after_snapshot(cluster): cluster.create(1) for _ in range(10): cluster.node(1).raft_exec('INCR', 'counter') info = cluster.node(1).raft_info() assert info['current_index'] == 11 # Make sure log is compacted assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' info = cluster.node(1).raft_info() assert info['log_entries'] == 0 assert info['current_index'] == 11 assert info['commit_index'] == 11
def test_reply_to_cache_invalidated_entry(cluster): """ Reply a RAFT redis command that have its entry already removed from the cache. """ cluster.create(3) assert cluster.leader == 1 # Configure a small cache assert cluster.node(1).raft_config_set('raft-log-max-cache-size', '1kb') # Break cluster to avoid commits cluster.node(2).terminate() cluster.node(3).terminate() # Send commands that are guarnateed to overflow the cache conns = [] for i in range(10): conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'SET', 'key%s' % i, 'x' * 1000) conns.append(conn) # give periodic job time to handle cache time.sleep(0.5) # confirm all raft entries were created but some have been evicted # from cache already. info = cluster.node(1).raft_info() assert info['log_entries'] == 15 assert info['cache_entries'] < 10 # Repair cluster and wait cluster.node(2).start() cluster.node(3).start() cluster.node(1).wait_for_num_voting_nodes(3) time.sleep(1) assert cluster.node(1).commit_index() == 15 # Expect TIMEOUT or OK for all for conn in conns: assert conn.can_read(timeout=1) try: assert conn.read_response() == b'OK' except ResponseError as err: assert str(err).startswith('TIMEOUT')
def test_stale_reads_on_leader_election(cluster): """ """ cluster.create(3) # Try 10 times for _ in range(10): val_written = cluster.raft_exec("INCR", "counter-1") leader = cluster.node(cluster.leader) leader.terminate() leader.start(verify=False) val_read = cluster.raft_exec('GET', 'counter-1') assert val_read is not None assert val_written == int(val_read) time.sleep(1)
def test_log_rollback(cluster): """ Rollback of log entries that were written in the minority. """ cluster.create(3) assert cluster.leader == 1 assert cluster.raft_exec('INCRBY', 'key', '111') == 111 # Break cluster cluster.node(2).terminate() cluster.node(3).terminate() # Load a command which can't be committed assert cluster.node(1).current_index() == 6 conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'INCRBY', 'key', '222') assert cluster.node(1).current_index() == 7 cluster.node(1).terminate() # We want to be sure the last entry is in the log log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count() == 7 # Restart the cluster without node 1, make sure the write was # not committed. cluster.node(2).start() cluster.node(3).start() cluster.node(2).wait_for_election() assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry # Restart node 1 cluster.node(1).start() cluster.node(1).wait_for_election() # Make another write and make sure it overwrites the previous one in # node 1's log assert cluster.raft_exec('INCRBY', 'key', '333') == 444 cluster.wait_for_unanimity() # Make sure log reflects the change log.reset() log.read() assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
def test_new_uncommitted_during_rewrite(cluster): cluster.create(3) # Take down majority to create uncommitted entries and check rewrite cluster.node(1).raft_exec('SET', 'key', '1') # Initiate compaction and wait to see it's in progress conn = cluster.node(1).client.connection_pool.get_connection('COMPACT') conn.send_command('RAFT.DEBUG', 'COMPACT', '2') cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes') assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes' # Send a bunch of writes cluster.node(1).raft_exec('INCRBY', 'key', '2') cluster.node(1).raft_exec('INCRBY', 'key', '3') cluster.node(1).raft_exec('INCRBY', 'key', '4') # Wait for compaction to end assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes' cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no') # Make sure our writes made it to the log log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 3 # Extra check -- Make sure we can read it back. Note that we need to start # all nodes because we don't log the commit index. cluster.node(1).terminate() cluster.node(2).terminate() cluster.node(3).terminate() cluster.node(1).start() cluster.node(2).start() cluster.node(3).start() cluster.node(1).wait_for_info_param('state', 'up') # Make sure cluster state is as expected assert cluster.raft_exec('get', 'key') == b'10' # Make sure node 1 state is as expected cluster.node(1).wait_for_log_applied() assert cluster.node(1).client.get('key') == b'10'
def test_all_committed_log_rewrite(cluster): """ Log rewrite operation when all entries are committed, so we expect an empty log. """ cluster.create(3) cluster.node(1).raft_exec('SET', 'key1', 'value') cluster.node(1).raft_exec('SET', 'key2', 'value') cluster.node(1).raft_exec('SET', 'key3', 'value') cluster.wait_for_unanimity() assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' assert cluster.node(1).raft_info()['log_entries'] == 0 # Make sure we have no log entries! log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 0
def test_multi_exec_proxying(cluster): """ Proxy a MULTI/EXEC sequence """ cluster.create(3) assert cluster.leader == 1 assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET', 'follower-proxy', 'yes') == b'OK' # Basic sanity n2 = cluster.node(2) assert n2.raft_info()['current_index'] == 5 assert n2.raft_exec('MULTI') assert n2.raft_exec('INCR', 'key') == b'QUEUED' assert n2.raft_exec('INCR', 'key') == b'QUEUED' assert n2.raft_exec('INCR', 'key') == b'QUEUED' assert n2.raft_exec('EXEC') == [1, 2, 3] assert n2.raft_info()['current_index'] == 6
def test_stale_reads_on_restarts(cluster, workload): """ Test proxy mode with MULTI transactions safety checks and reconnections (dropping clients with CLIENT KILL). """ thread_count = 50 cycles = 20 cluster.create(3, raft_args={ 'follower-proxy': 'yes', 'raftize-all-commands': 'yes' }) workload.start(thread_count, cluster, MonotonicIncrCheck) for _ in range(cycles): time.sleep(1) cluster.restart() logging.info('All cycles finished') workload.stop()
def test_node_history_with_same_address(cluster): "" "" cluster.create(5) cluster.raft_exec("INCR", "step-counter") # Remove nodes ports = [] for node_id in [2, 3, 4, 5]: ports.append(cluster.node(node_id).port) cluster.remove_node(node_id) cluster.leader_node().wait_for_log_applied() cluster.node(cluster.leader).wait_for_num_nodes(1) # Now add and remove several more times for _ in range(5): for port in ports: n = cluster.add_node(port=port) cluster.leader_node().wait_for_num_nodes(2) cluster.leader_node().wait_for_log_applied() cluster.remove_node(n.id) cluster.leader_node().wait_for_num_nodes(1) cluster.leader_node().wait_for_log_applied() # Add enough data in the log to satisfy timing for _ in range(3000): cluster.raft_exec("INCR", "step-counter") # Add another node new_node = cluster.add_node(port=ports[0]) new_node.wait_for_node_voting() # Terminate all cluster.terminate() # Start new node cluster.start() # need some time to start applying logs.. time.sleep(2) assert cluster.raft_exec("GET", "step-counter") == b'3001'
def test_fuzzing_with_restarts(cluster): """ Basic Raft fuzzer test """ nodes = 3 cycles = 100 cluster.create(nodes) for i in range(cycles): assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1 logging.info('---------- Executed INCRBY # %s', i) if i % 7 == 0: r = random.randint(1, nodes) logging.info('********** Restarting node %s **********', r) cluster.node(r).restart() cluster.node(r).wait_for_election() logging.info('********** Node %s is UP **********', r) assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_fuzzing_with_config_changes(cluster): """ Basic Raft fuzzer test """ nodes = 5 cycles = 100 cluster.create(nodes) for i in range(cycles): assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1 if random.randint(1, 7) == 1: try: node_id = cluster.random_node_id() cluster.remove_node(node_id) except ResponseError: continue cluster.add_node().wait_for_node_voting() assert int(cluster.raft_exec('GET', 'counter')) == cycles
def test_single_voting_change_enforced(cluster): """ A single concurrent voting change is enforced when removing nodes. """ cluster.create(5) assert cluster.leader == 1 # Simulate a partition cluster.node(2).terminate() cluster.node(3).terminate() cluster.node(4).terminate() assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '5') == b'OK' with raises(ResponseError, match='a voting change is already in progress'): assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '4') == b'OK' time.sleep(1) assert cluster.node(1).raft_info()['num_nodes'] == 5