Пример #1
0
def test_fuzzing_with_proxy_multi_and_restarts(cluster, workload):
    """
    Test proxy with transaction safety and random node restarts.
    """

    nodes = 3
    cycles = 20
    thread_count = 200

    cluster.create(nodes,
                   raft_args={
                       'raftize-all-commands': 'yes',
                       'follower-proxy': 'yes'
                   })
    workload.start(thread_count, cluster, MultiWithLargeReply)
    for i in range(cycles):
        time.sleep(1)
        try:
            logging.info('Cycle %s: %s', i, workload.stats())
            cluster.random_node().restart()
        except ResponseError as err:
            logging.error('Remove node: %s', err)
            continue
    logging.info('All cycles finished')
    workload.stop()
Пример #2
0
def test_log_fixup_after_snapshot_delivery(cluster):
    """
    Log must be restarted when loading a snapshot.
    """

    cluster.create(3)
    cluster.node(1).raft_exec('INCR', 'key')
    cluster.node(1).raft_exec('INCR', 'key')

    cluster.node(2).terminate()
    cluster.node(1).raft_exec('INCR', 'key')
    assert cluster.node(1).client.execute_command(
        'RAFT.DEBUG', 'COMPACT') == b'OK'

    # Confirm node 2's log starts from 1
    log = RaftLog(cluster.node(2).raftlog)
    log.read()
    assert log.header().snapshot_index() == 0

    cluster.node(2).start()
    cluster.node(2).wait_for_info_param('state', 'up')

    # node 2 must get a snapshot to sync, make sure this happens and that
    # the log is ok.
    cluster.node(2).wait_for_current_index(8)
    log = RaftLog(cluster.node(2).raftlog)
    log.read()
    assert log.header().snapshot_index() == 8
Пример #3
0
def test_full_cluster_remove(cluster):
    """
    Remove all cluster nodes.
    """

    cluster.create(5)

    leader = cluster.leader_node()
    expected_nodes = 5
    for node_id in (2, 3, 4, 5):
        leader.client.execute_command('RAFT.NODE', 'REMOVE', str(node_id))
        expected_nodes -= 1
        leader.wait_for_num_nodes(expected_nodes)

    # make sure other nodes are down
    for node_id in (2, 3, 4, 5):
        assert not cluster.node(node_id).process_is_up()

    # and make sure they start up in uninitialized state
    for node_id in (2, 3, 4, 5):
        cluster.node(node_id).terminate()
        cluster.node(node_id).start()

    for node_id in (2, 3, 4, 5):
        assert cluster.node(node_id).raft_info()['state'] == 'uninitialized'
Пример #4
0
def test_fuzzing_with_restarts_and_rewrites(cluster):
    """
    Counter fuzzer with log rewrites.
    """

    nodes = 3
    cycles = 100

    cluster.create(nodes)
    # Randomize max log entries
    for node in cluster.nodes.values():
        node.client.execute_command('RAFT.CONFIG', 'SET',
                                    'raft-log-max-file-size',
                                    str(random.randint(1000, 2000)))

    for i in range(cycles):
        assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
        logging.info('---------- Executed INCRBY # %s', i)
        if random.randint(1, 7) == 1:
            r = random.randint(1, nodes)
            logging.info('********** Restarting node %s **********', r)
            cluster.node(r).restart()
            cluster.node(r).wait_for_election()
            logging.info('********** Node %s is UP **********', r)

    assert int(cluster.raft_exec('GET', 'counter')) == cycles
Пример #5
0
def test_rolled_back_read_only_multi_reply(cluster):
    """
    Watch the reply to a MULTI operation that ends up being discarded
    due to election change before commit.
    """

    cluster.create(3)
    cluster.node(1).raft_exec('SET', 'key', 'value')

    cluster.node(2).pause()
    cluster.node(3).pause()

    conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
    conn.send_command('RAFT', 'MULTI')
    assert conn.read_response() == b'OK'
    conn.send_command('RAFT', 'GET', 'key')
    assert conn.read_response() == b'QUEUED'
    conn.send_command('RAFT', 'EXEC')

    cluster.node(1).pause()

    cluster.node(2).kill()
    cluster.node(3).kill()
    cluster.node(2).start()
    cluster.node(3).start()

    cluster.node(2).wait_for_election()
    cluster.node(1).resume()

    with raises(ResponseError, match='TIMEOUT'):
        assert conn.read_response() == None
Пример #6
0
def test_proxying(cluster):
    """
    Command proxying from follower to leader works
    """
    cluster.create(3)
    assert cluster.leader == 1
    with raises(ResponseError, match='MOVED'):
        assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK'
    assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET',
                                                  'follower-proxy',
                                                  'yes') == b'OK'

    # Basic sanity
    assert cluster.node(2).raft_exec('SET', 'key', 'value') == b'OK'
    assert cluster.raft_exec('GET', 'key') == b'value'

    # Numeric values
    assert cluster.node(2).raft_exec('SADD', 'myset', 'a') == 1
    assert cluster.node(2).raft_exec('SADD', 'myset', 'b') == 1
    # Multibulk
    assert set(cluster.node(2).raft_exec('SMEMBERS',
                                         'myset')) == set([b'a', b'b'])
    # Nested multibulk
    assert set(
        cluster.node(2).raft_exec('EVAL', 'return {{\'a\',\'b\',\'c\'}};',
                                  0)[0]) == set([b'a', b'b', b'c'])
    # Error
    with raises(ResponseError, match='WRONGTYPE'):
        cluster.node(2).raft_exec('INCR', 'myset')
Пример #7
0
def test_stability_with_snapshots_and_restarts(cluster, workload):
    """
    Test stability of the cluster with frequent snapshoting.
    """

    thread_count = 100
    duration = 300

    cluster.create(5,
                   raft_args={
                       'follower-proxy': 'yes',
                       'raftize-all-commands': 'yes',
                       'raft-log-max-file-size': '2000'
                   })

    workload.start(thread_count, cluster, MultiWithLargeReply)

    # Monitor progress
    start = time.time()
    last_commit_index = 0
    while start + duration > time.time():
        time.sleep(2)
        cluster.random_node().restart()

    workload.stop()
Пример #8
0
def test_readonly_commands(cluster):
    """
    Test read-only command execution, which does not go through the Raft
    log.
    """
    cluster.create(3)
    assert cluster.leader == 1

    # Write something
    assert cluster.node(1).current_index() == 5
    assert cluster.node(1).raft_exec('SET', 'key', 'value') == b'OK'
    assert cluster.node(1).current_index() == 6

    # Read something, log should not grow
    assert cluster.node(1).raft_exec('GET', 'key') == b'value'
    assert cluster.node(1).current_index() == 6

    # Tear down cluster, reads should hang
    cluster.node(2).terminate()
    cluster.node(3).terminate()
    conn = cluster.node(1).client.connection_pool.get_connection(
        'RAFT', socket_timeout=1)
    conn.send_command('RAFT', 'GET', 'key')
    assert not conn.can_read(timeout=1)

    # Now configure non-quorum reads
    cluster.node(1).raft_config_set('quorum-reads', 'no')
    assert cluster.node(1).raft_exec('GET', 'key') == b'value'
Пример #9
0
def test_proxy_stability_under_load(cluster, workload):
    """
    Test stability of the cluster with follower proxy under load.
    """

    thread_count = 500
    duration = 300

    cluster.create(5,
                   raft_args={
                       'follower-proxy': 'yes',
                       'raftize-all-commands': 'yes'
                   })

    workload.start(thread_count, cluster, MultiWithLargeReply)

    # Monitor progress
    start = time.time()
    last_commit_index = 0
    while start + duration > time.time():
        time.sleep(2)
        new_commit_index = cluster.node(cluster.leader).commit_index()
        assert new_commit_index > last_commit_index
        last_commit_index = new_commit_index

    workload.stop()
Пример #10
0
def test_uncommitted_log_rewrite(cluster):
    cluster.create(3)

    # Log contains 5 config entries

    # Take down majority to create uncommitted entries and check rewrite
    cluster.node(1).raft_exec('SET', 'key', 'value')  # Entry idx 6
    cluster.node(2).terminate()
    cluster.node(3).terminate()
    conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
    conn.send_command('RAFT', 'SET', 'key2', 'value2')  # Entry idx 7

    assert cluster.node(1).current_index() == 7
    assert cluster.node(1).commit_index() == 6
    assert cluster.node(1).client.execute_command(
        'RAFT.DEBUG', 'COMPACT') == b'OK'
    assert cluster.node(1).raft_info()['log_entries'] == 1

    log = RaftLog(cluster.node(1).raftlog)
    log.read()
    assert log.entry_count(LogEntry.LogType.NORMAL) == 1

    cluster.node(1).kill()
    cluster.node(1).start()
    cluster.node(1).wait_for_info_param('state', 'up')
    assert cluster.node(1).current_index() == 7
    assert cluster.node(1).commit_index() == 6
    assert cluster.node(1).raft_info()['log_entries'] == 1
Пример #11
0
def test_cfg_node_removed_from_snapshot(cluster):
    """
    Node able to learn that another node left by reading the snapshot metadata.
    """
    cluster.create(5)
    cluster.node(1).raft_exec('SET', 'key', 'value')
    cluster.wait_for_unanimity()

    # interrupt
    # we now take down node 4 so it doesn't get updates and remove node 5.
    cluster.node(4).terminate()
    cluster.remove_node(5)
    cluster.wait_for_unanimity(exclude=[4])
    cluster.node(1).wait_for_log_applied()
    assert cluster.node(1).raft_info()['num_nodes'] == 4

    # now compact logs
    cluster.wait_for_unanimity(exclude=[4])
    assert cluster.node(1).client.execute_command(
        'RAFT.DEBUG', 'COMPACT') == b'OK'
    assert cluster.node(1).raft_info()['log_entries'] == 0

    # bring back node 4
    cluster.node(4).start()
    cluster.node(4).wait_for_election()
    cluster.wait_for_unanimity()
    assert cluster.node(4).raft_info()['num_nodes'] == 4
Пример #12
0
def test_config_from_second_generation_snapshot(cluster):
    """
    A regression test for #44: confirm that if we load a snapshot
    on startup, do nothing, then re-create a snapshot we don't end
    up with a messed up nodes config.
    """
    cluster.create(3)

    # Bump the log a bit
    for _ in range(20):
        assert cluster.raft_exec('INCR', 'testkey')

    # Compact to get rid of logs
    node3 = cluster.node(3)
    assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'

    # Restart node
    node3.restart()
    node3.wait_for_node_voting()

    # Bump the log a bit
    for _ in range(20):
        assert cluster.raft_exec('INCR', 'testkey')

    # Recompact
    cluster.wait_for_unanimity()
    assert node3.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'

    node3.restart()
    node3.wait_for_node_voting()

    assert node3.raft_info()['num_nodes'] == 3
Пример #13
0
def test_removed_node_remains_dead(cluster):
    """
    A removed node stays down and does not resurrect in any case.
    """

    cluster.create(3)

    # Some baseline data
    for _ in range(100):
        cluster.raft_exec('INCR', 'counter')

    # Remove node 3
    cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '3')
    cluster.node(1).wait_for_num_voting_nodes(2)

    # Add more data
    for _ in range(100):
        cluster.raft_exec('INCR', 'counter')

    # Check
    node = cluster.node(3)

    # Verify node 3 does not accept writes
    with raises(RedisError):
        node.client.execute_command('RAFT', 'INCR', 'counter')

    # Verify node 3 still does not accept writes after a restart
    node.terminate()
    node.start()

    with raises(RedisError):
        node.client.execute_command('RAFT', 'INCR', 'counter')
Пример #14
0
def test_snapshot_delivery(cluster):
    """
    Ability to properly deliver and load a snapshot.
    """

    cluster.create(3, raft_args={'raftize-all-commands': 'yes'})
    n1 = cluster.node(1)
    n1.raft_exec('INCR', 'testkey')
    n1.raft_exec('INCR', 'testkey')
    n1.raft_exec('INCR', 'testkey')
    for i in range(1000):
        pipe = n1.client.pipeline(transaction=True)
        for j in range(100):
            pipe.rpush('list-%s' % i, 'elem-%s' % j)
        pipe.execute()
    cluster.node(3).terminate()
    n1.raft_exec('SETRANGE', 'bigkey', '104857600', 'x')
    n1.raft_exec('INCR', 'testkey')
    assert n1.client.get('testkey') == b'4'

    assert n1.client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'
    assert n1.raft_info()['log_entries'] == 0

    n3 = cluster.node(3)
    n3.start()
    n1.raft_exec('INCR', 'testkey')

    n3.wait_for_node_voting()
    cluster.wait_for_unanimity()
    n3.wait_for_log_applied()

    n3.client.execute_command('RAFT.CONFIG', 'SET',
                              'raftize-all-commands', 'no')
    assert n3.client.get('testkey') == b'5'
Пример #15
0
def test_rolled_back_reply(cluster):
    """
    Watch the reply to a write operation that ends up being discarded
    due to election change before commit.
    """

    cluster.create(3)

    cluster.node(2).pause()
    cluster.node(3).pause()

    conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
    conn.send_command('RAFT', 'INCR', 'key')

    cluster.node(1).pause()

    cluster.node(2).kill()
    cluster.node(3).kill()
    cluster.node(2).start()
    cluster.node(3).start()

    cluster.node(2).wait_for_election()
    cluster.node(1).resume()

    with raises(ResponseError, match='TIMEOUT'):
        conn.read_response()
Пример #16
0
def test_leader_removal_not_allowed(cluster):
    """
    Leader node cannot be removed.
    """

    cluster.create(3)
    assert cluster.leader == 1
    with raises(ResponseError, match='cannot remove leader'):
        cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE', '1')
Пример #17
0
def test_update_self_voting_state_from_snapshot(cluster):
    cluster.create(3)

    assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'NODECFG', '2', '-voting') == b'OK'
    assert cluster.node(2).raft_info()['is_voting'] == 'yes'
    assert cluster.node(1).client.execute_command('RAFT.DEBUG', 'COMPACT') == b'OK'

    cluster.node(1).client.execute_command('RAFT.DEBUG', 'SENDSNAPSHOT', '2')
    cluster.node(2).wait_for_info_param('snapshots_loaded', 1)
    assert cluster.node(2).raft_info()['is_voting'] == 'no'
Пример #18
0
def test_reelection_basic_flow(cluster):
    """
    Basic reelection flow
    """
    cluster.create(3)
    assert cluster.leader == 1
    assert cluster.raft_exec('SET', 'key', 'value') == b'OK'
    cluster.node(1).terminate()
    cluster.node(2).wait_for_election()
    assert cluster.raft_exec('SET', 'key2', 'value2') == b'OK'
    cluster.exec_all('GET', 'key2')
Пример #19
0
def test_index_correct_right_after_snapshot(cluster):
    cluster.create(1)
    for _ in range(10):
        cluster.node(1).raft_exec('INCR', 'counter')
    info = cluster.node(1).raft_info()
    assert info['current_index'] == 11

    # Make sure log is compacted
    assert cluster.node(1).client.execute_command(
        'RAFT.DEBUG', 'COMPACT') == b'OK'
    info = cluster.node(1).raft_info()
    assert info['log_entries'] == 0
    assert info['current_index'] == 11
    assert info['commit_index'] == 11
Пример #20
0
def test_reply_to_cache_invalidated_entry(cluster):
    """
    Reply a RAFT redis command that have its entry already removed
    from the cache.
    """

    cluster.create(3)
    assert cluster.leader == 1

    # Configure a small cache
    assert cluster.node(1).raft_config_set('raft-log-max-cache-size', '1kb')

    # Break cluster to avoid commits
    cluster.node(2).terminate()
    cluster.node(3).terminate()

    # Send commands that are guarnateed to overflow the cache
    conns = []
    for i in range(10):
        conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
        conn.send_command('RAFT', 'SET', 'key%s' % i, 'x' * 1000)
        conns.append(conn)

    # give periodic job time to handle cache
    time.sleep(0.5)

    # confirm all raft entries were created but some have been evicted
    # from cache already.
    info = cluster.node(1).raft_info()
    assert info['log_entries'] == 15
    assert info['cache_entries'] < 10

    # Repair cluster and wait
    cluster.node(2).start()
    cluster.node(3).start()
    cluster.node(1).wait_for_num_voting_nodes(3)
    time.sleep(1)
    assert cluster.node(1).commit_index() == 15

    # Expect TIMEOUT or OK for all
    for conn in conns:
        assert conn.can_read(timeout=1)
        try:
            assert conn.read_response() == b'OK'
        except ResponseError as err:
            assert str(err).startswith('TIMEOUT')
Пример #21
0
def test_stale_reads_on_leader_election(cluster):
    """
    """
    cluster.create(3)

    # Try 10 times
    for _ in range(10):
        val_written = cluster.raft_exec("INCR", "counter-1")

        leader = cluster.node(cluster.leader)
        leader.terminate()
        leader.start(verify=False)

        val_read = cluster.raft_exec('GET', 'counter-1')
        assert val_read is not None
        assert val_written == int(val_read)
        time.sleep(1)
Пример #22
0
def test_log_rollback(cluster):
    """
    Rollback of log entries that were written in the minority.
    """

    cluster.create(3)
    assert cluster.leader == 1
    assert cluster.raft_exec('INCRBY', 'key', '111') == 111

    # Break cluster
    cluster.node(2).terminate()
    cluster.node(3).terminate()

    # Load a command which can't be committed
    assert cluster.node(1).current_index() == 6
    conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
    conn.send_command('RAFT', 'INCRBY', 'key', '222')
    assert cluster.node(1).current_index() == 7
    cluster.node(1).terminate()

    # We want to be sure the last entry is in the log
    log = RaftLog(cluster.node(1).raftlog)
    log.read()
    assert log.entry_count() == 7

    # Restart the cluster without node 1, make sure the write was
    # not committed.
    cluster.node(2).start()
    cluster.node(3).start()
    cluster.node(2).wait_for_election()
    assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry

    # Restart node 1
    cluster.node(1).start()
    cluster.node(1).wait_for_election()

    # Make another write and make sure it overwrites the previous one in
    # node 1's log
    assert cluster.raft_exec('INCRBY', 'key', '333') == 444
    cluster.wait_for_unanimity()

    # Make sure log reflects the change
    log.reset()
    log.read()
    assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
Пример #23
0
def test_new_uncommitted_during_rewrite(cluster):
    cluster.create(3)

    # Take down majority to create uncommitted entries and check rewrite
    cluster.node(1).raft_exec('SET', 'key', '1')

    # Initiate compaction and wait to see it's in progress
    conn = cluster.node(1).client.connection_pool.get_connection('COMPACT')
    conn.send_command('RAFT.DEBUG', 'COMPACT', '2')
    cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes')
    assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'

    # Send a bunch of writes
    cluster.node(1).raft_exec('INCRBY', 'key', '2')
    cluster.node(1).raft_exec('INCRBY', 'key', '3')
    cluster.node(1).raft_exec('INCRBY', 'key', '4')

    # Wait for compaction to end
    assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
    cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no')

    # Make sure our writes made it to the log
    log = RaftLog(cluster.node(1).raftlog)
    log.read()
    assert log.entry_count(LogEntry.LogType.NORMAL) == 3

    # Extra check -- Make sure we can read it back. Note that we need to start
    # all nodes because we don't log the commit index.
    cluster.node(1).terminate()
    cluster.node(2).terminate()
    cluster.node(3).terminate()
    cluster.node(1).start()
    cluster.node(2).start()
    cluster.node(3).start()

    cluster.node(1).wait_for_info_param('state', 'up')

    # Make sure cluster state is as expected
    assert cluster.raft_exec('get', 'key') == b'10'

    # Make sure node 1 state is as expected
    cluster.node(1).wait_for_log_applied()
    assert cluster.node(1).client.get('key') == b'10'
Пример #24
0
def test_all_committed_log_rewrite(cluster):
    """
    Log rewrite operation when all entries are committed, so we expect an
    empty log.
    """
    cluster.create(3)
    cluster.node(1).raft_exec('SET', 'key1', 'value')
    cluster.node(1).raft_exec('SET', 'key2', 'value')
    cluster.node(1).raft_exec('SET', 'key3', 'value')
    cluster.wait_for_unanimity()

    assert cluster.node(1).client.execute_command(
        'RAFT.DEBUG', 'COMPACT') == b'OK'
    assert cluster.node(1).raft_info()['log_entries'] == 0

    # Make sure we have no log entries!
    log = RaftLog(cluster.node(1).raftlog)
    log.read()
    assert log.entry_count(LogEntry.LogType.NORMAL) == 0
Пример #25
0
def test_multi_exec_proxying(cluster):
    """
    Proxy a MULTI/EXEC sequence
    """
    cluster.create(3)
    assert cluster.leader == 1
    assert cluster.node(2).client.execute_command('RAFT.CONFIG', 'SET',
                                                  'follower-proxy',
                                                  'yes') == b'OK'

    # Basic sanity
    n2 = cluster.node(2)
    assert n2.raft_info()['current_index'] == 5
    assert n2.raft_exec('MULTI')
    assert n2.raft_exec('INCR', 'key') == b'QUEUED'
    assert n2.raft_exec('INCR', 'key') == b'QUEUED'
    assert n2.raft_exec('INCR', 'key') == b'QUEUED'
    assert n2.raft_exec('EXEC') == [1, 2, 3]
    assert n2.raft_info()['current_index'] == 6
Пример #26
0
def test_stale_reads_on_restarts(cluster, workload):
    """
    Test proxy mode with MULTI transactions safety checks and
    reconnections (dropping clients with CLIENT KILL).
    """

    thread_count = 50
    cycles = 20
    cluster.create(3,
                   raft_args={
                       'follower-proxy': 'yes',
                       'raftize-all-commands': 'yes'
                   })
    workload.start(thread_count, cluster, MonotonicIncrCheck)
    for _ in range(cycles):
        time.sleep(1)
        cluster.restart()
    logging.info('All cycles finished')
    workload.stop()
Пример #27
0
def test_node_history_with_same_address(cluster):
    ""
    ""
    cluster.create(5)
    cluster.raft_exec("INCR", "step-counter")

    # Remove nodes
    ports = []
    for node_id in [2, 3, 4, 5]:
        ports.append(cluster.node(node_id).port)
        cluster.remove_node(node_id)
        cluster.leader_node().wait_for_log_applied()
    cluster.node(cluster.leader).wait_for_num_nodes(1)

    # Now add and remove several more times
    for _ in range(5):
        for port in ports:
            n = cluster.add_node(port=port)
            cluster.leader_node().wait_for_num_nodes(2)
            cluster.leader_node().wait_for_log_applied()
            cluster.remove_node(n.id)
            cluster.leader_node().wait_for_num_nodes(1)
            cluster.leader_node().wait_for_log_applied()

    # Add enough data in the log to satisfy timing
    for _ in range(3000):
        cluster.raft_exec("INCR", "step-counter")

    # Add another node
    new_node = cluster.add_node(port=ports[0])
    new_node.wait_for_node_voting()

    # Terminate all
    cluster.terminate()

    # Start new node
    cluster.start()

    # need some time to start applying logs..
    time.sleep(2)

    assert cluster.raft_exec("GET", "step-counter") == b'3001'
Пример #28
0
def test_fuzzing_with_restarts(cluster):
    """
    Basic Raft fuzzer test
    """

    nodes = 3
    cycles = 100

    cluster.create(nodes)
    for i in range(cycles):
        assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
        logging.info('---------- Executed INCRBY # %s', i)
        if i % 7 == 0:
            r = random.randint(1, nodes)
            logging.info('********** Restarting node %s **********', r)
            cluster.node(r).restart()
            cluster.node(r).wait_for_election()
            logging.info('********** Node %s is UP **********', r)

    assert int(cluster.raft_exec('GET', 'counter')) == cycles
Пример #29
0
def test_fuzzing_with_config_changes(cluster):
    """
    Basic Raft fuzzer test
    """

    nodes = 5
    cycles = 100

    cluster.create(nodes)
    for i in range(cycles):
        assert cluster.raft_exec('INCRBY', 'counter', 1) == i + 1
        if random.randint(1, 7) == 1:
            try:
                node_id = cluster.random_node_id()
                cluster.remove_node(node_id)
            except ResponseError:
                continue
            cluster.add_node().wait_for_node_voting()

    assert int(cluster.raft_exec('GET', 'counter')) == cycles
Пример #30
0
def test_single_voting_change_enforced(cluster):
    """
    A single concurrent voting change is enforced when removing nodes.
    """

    cluster.create(5)
    assert cluster.leader == 1

    # Simulate a partition
    cluster.node(2).terminate()
    cluster.node(3).terminate()
    cluster.node(4).terminate()

    assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE',
                                                  '5') == b'OK'
    with raises(ResponseError, match='a voting change is already in progress'):
        assert cluster.node(1).client.execute_command('RAFT.NODE', 'REMOVE',
                                                      '4') == b'OK'

    time.sleep(1)
    assert cluster.node(1).raft_info()['num_nodes'] == 5