def test_fuzzing_with_proxy_multi_and_restarts(cluster, workload):
"""
Test proxy with transaction safety and random node restarts.
"""
nodes = 3
cycles = 20
thread_count = 200
cluster.create(nodes,
raft_args={
'raftize-all-commands': 'yes',
'follower-proxy': 'yes'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
for i in range(cycles):
time.sleep(1)
try:
logging.info('Cycle %s: %s', i, workload.stats())
cluster.random_node().restart()
except ResponseError as err:
logging.error('Remove node: %s', err)
continue
logging.info('All cycles finished')
workload.stop()
def test_stability_with_snapshots_and_restarts(cluster, workload):
"""
Test stability of the cluster with frequent snapshoting.
"""
thread_count = 100
duration = 300
cluster.create(5,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes',
'raft-log-max-file-size': '2000'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
# Monitor progress
start = time.time()
last_commit_index = 0
while start + duration > time.time():
time.sleep(2)
cluster.random_node().restart()
workload.stop()
def test_proxy_stability_under_load(cluster, workload):
"""
Test stability of the cluster with follower proxy under load.
"""
thread_count = 500
duration = 300
cluster.create(5,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
# Monitor progress
start = time.time()
last_commit_index = 0
while start + duration > time.time():
time.sleep(2)
new_commit_index = cluster.node(cluster.leader).commit_index()
assert new_commit_index > last_commit_index
last_commit_index = new_commit_index
workload.stop()
def test_stale_reads_on_restarts(cluster, workload):
"""
Test proxy mode with MULTI transactions safety checks and
reconnections (dropping clients with CLIENT KILL).
"""
thread_count = 50
cycles = 20
cluster.create(3,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes'
})
workload.start(thread_count, cluster, MonotonicIncrCheck)
for _ in range(cycles):
time.sleep(1)
cluster.restart()
logging.info('All cycles finished')
workload.stop()
def test_proxy_with_multi_and_reconnections(cluster, workload):
"""
Test proxy mode with MULTI transactions safety checks and
reconnections (dropping clients with CLIENT KILL).
"""
thread_count = 100
cycles = 20
cluster.create(3,
raft_args={
'follower-proxy': 'yes',
'raftize-all-commands': 'yes'
})
workload.start(thread_count, cluster, MultiWithLargeReply)
for _ in range(cycles):
time.sleep(1)
logging.info('Initiating client kill cycle')
cluster.leader_node().client.execute_command('CLIENT', 'KILL', 'TYPE',
'normal')
logging.info('All cycles finished')
workload.stop()