def test_future_primaries_replicas_increase(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards, tdir, tconf, allPluginsPath):
# Don't delete NodeStates, so we could check them.
global old_commit
old_commit = txnPoolNodeSet[0].write_manager.future_primary_handler.commit_batch
for node in txnPoolNodeSet:
node.write_manager.future_primary_handler.commit_batch = lambda three_pc_batch, prev_handler_result=None: 0
initial_primaries = copy.copy(txnPoolNodeSet[0].primaries)
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
# Increase replicas count
add_new_node(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], tdir, tconf, allPluginsPath)
new_view_no = checkViewNoForNodes(txnPoolNodeSet)
assert new_view_no == starting_view_number + 1
# "seq_no + 2" because 1 domain and 1 pool txn.
node = txnPoolNodeSet[0]
with delay_rules(node.nodeIbStasher, cDelay()):
req = sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_stewards[0], 1)[0][0]
req = Request(**req)
three_pc_batch = ThreePcBatch(DOMAIN_LEDGER_ID, 0, 0, 1, time.time(),
randomString(),
randomString(),
['a', 'b', 'c'], [req.digest], pp_digest='')
primaries = node.write_manager.future_primary_handler.post_batch_applied(three_pc_batch)
assert len(primaries) == len(initial_primaries) + 1
assert len(primaries) == len(node.primaries)
def test_future_primaries_replicas_increase(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_stewards, tdir, tconf,
allPluginsPath):
# Don't delete NodeStates, so we could check them.
global old_commit
old_commit = txnPoolNodeSet[
0].write_manager.future_primary_handler.commit_batch
for node in txnPoolNodeSet:
node.write_manager.future_primary_handler.commit_batch = lambda three_pc_batch, prev_handler_result=None: 0
initial_primaries = copy.copy(txnPoolNodeSet[0].primaries)
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
# Increase replicas count
add_new_node(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], tdir, tconf, allPluginsPath)
new_view_no = checkViewNoForNodes(txnPoolNodeSet)
assert new_view_no == starting_view_number + 1
# "seq_no + 2" because 1 domain and 1 pool txn.
state = txnPoolNodeSet[0].write_manager.future_primary_handler.node_states[
-1]
assert len(state.primaries) == len(initial_primaries) + 1
assert len(state.primaries) == len(txnPoolNodeSet[0].primaries)
def test_audit_multiple_uncommitted_node_regs(looper, tdir, tconf,
allPluginsPath, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
sdk_wallet_steward):
'''
- Delay COMMITS on 1 Node
- Add 2 more nodes (so that the slow node hs multiple uncommitted node txns)
- Make sure that all nodes have equal state eventually
'''
slow_node = txnPoolNodeSet[-1]
fast_nodes = [node for node in txnPoolNodeSet if node != slow_node]
slow_stashers = [slow_node.nodeIbStasher]
# let's ignore view changes for simplicity here
start_delaying([n.nodeIbStasher for n in txnPoolNodeSet], icDelay())
with delay_rules(slow_stashers, cDelay()):
# Add Node5
new_node = add_new_node(looper,
fast_nodes,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath,
name='Psi',
wait_till_added=False)
txnPoolNodeSet.append(new_node)
start_delaying(new_node.nodeIbStasher, icDelay())
# Add Node6
new_node = add_new_node(looper,
fast_nodes,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath,
name='Eta',
wait_till_added=False)
txnPoolNodeSet.append(new_node)
start_delaying(new_node.nodeIbStasher, icDelay())
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=20)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_set_H_greater_then_last_ppseqno(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
# send LOG_SIZE requests and check, that all watermarks on all replicas is not changed
# and now is (0, LOG_SIZE)
"""Send random requests for moving watermarks"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, LOG_SIZE)
# check, that all of node set up watermark greater, then default and
# ppSeqNo with number LOG_SIZE + 1 will be out from default watermark
assert txnPoolNodeSet[0].replicas[1].last_ordered_3pc[1] > 0
looper.runFor(30)
assert txnPoolNodeSet[0].replicas[1].last_ordered_3pc[1] == LOG_SIZE
for n in txnPoolNodeSet:
for r in n.replicas._replicas.values():
assert r.h >= LOG_SIZE
assert r.H >= LOG_SIZE + LOG_SIZE
"""Adding new node, for scheduling propagate primary procedure"""
new_node = add_new_node(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, tdir, tconf, allPluginsPath)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet,
exclude_from_check=['check_last_ordered_3pc_backup'])
"""Check, that backup replicas set watermark as (0, maxInt)"""
# Check, replica.h is set from last_ordered_3PC and replica.H is set to maxsize
for r in new_node.replicas.values():
assert r.h == r.last_ordered_3pc[1]
if r.isMaster:
assert r.H == r.last_ordered_3pc[1] + LOG_SIZE
else:
assert r.H == sys.maxsize
"""Send requests and check. that backup replicas does not stashing it by outside watermarks reason"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)
# check, that there is no any stashed "outside watermark" messages.
for r in new_node.replicas.values():
assert r.stasher.stash_size(STASH_WATERMARKS) == 0
"""Force view change and check, that all backup replicas setup H as a default<
(not propagate primary logic)"""
"""This need to ensure, that next view_change does not break watermark setting logic"""
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper, txnPoolNodeSet)
for r in new_node.replicas.values():
if not r.isMaster:
assert r.h == 0
assert r.H == LOG_SIZE
def test_set_H_greater_then_last_ppseqno(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
# send LOG_SIZE requests and check, that all watermarks on all replicas is not changed
# and now is (0, LOG_SIZE)
"""Send random requests for moving watermarks"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, LOG_SIZE)
# check, that all of node set up watermark greater, then default and
# ppSeqNo with number LOG_SIZE + 1 will be out from default watermark
for n in txnPoolNodeSet:
for r in n.replicas._replicas.values():
assert r.h >= LOG_SIZE
assert r.H >= LOG_SIZE + LOG_SIZE
"""Adding new node, for scheduling propagate primary procedure"""
new_node = add_new_node(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, tdir, tconf, allPluginsPath)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
"""Check, that backup replicas set watermark as (0, maxInt)"""
# Check, replica.h is set from last_ordered_3PC and replica.H is set to maxsize
for r in new_node.replicas.values():
assert r.h == r.last_ordered_3pc[1]
if r.isMaster:
assert r.H == r.last_ordered_3pc[1] + LOG_SIZE
else:
assert r.H == sys.maxsize
"""Send requests and check. that backup replicas does not stashing it by outside watermarks reason"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)
# check, that there is no any stashed "outside watermark" messages.
for r in new_node.replicas.values():
assert len(r.stashingWhileOutsideWaterMarks) == 0
"""Force view change and check, that all backup replicas setup H as a default
(not propagate primary logic)"""
"""This need to ensure, that next view_change does not break watermark setting logic"""
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper, txnPoolNodeSet)
for r in new_node.replicas.values():
if not r.isMaster:
assert r.h == 0
assert r.H == LOG_SIZE
def check_view_change_adding_new_node(looper,
tdir,
tconf,
allPluginsPath,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
sdk_wallet_steward,
slow_nodes=[],
delay_commit=False,
delay_pre_prepare=False):
# Pre-requisites: viewNo=3, Primary is Node4
for viewNo in range(1, 4):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, viewNo)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
# Delay 3PC messages on slow nodes
fast_nodes = [node for node in txnPoolNodeSet if node not in slow_nodes]
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
delayers = []
if delay_pre_prepare:
delayers.append(ppDelay())
delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE]))
if delay_commit:
delayers.append(cDelay())
with delay_rules_without_processing(slow_stashers, *delayers):
# Add Node5
new_node = add_new_node(looper, fast_nodes, sdk_pool_handle,
sdk_wallet_steward, tdir, tconf,
allPluginsPath)
old_set = list(txnPoolNodeSet)
txnPoolNodeSet.append(new_node)
# Trigger view change
trigger_view_change(txnPoolNodeSet)
# make sure view change is finished eventually
waitForViewChange(looper, old_set, 4)
ensureElectionsDone(looper, old_set)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_6th_node_join_after_view_change_by_master_restart(
looper, txnPoolNodeSet, tdir, tconf, allPluginsPath, sdk_pool_handle,
sdk_wallet_steward, limitTestRunningTime):
"""
Test steps:
1. start pool of 4 nodes
2. force 4 view change by restarting primary node
3. now primary node must be Alpha, then add new node, named Epsilon
4. ensure, that Epsilon was added and catch-up done
5. send some txns
6. force 4 view change. Now primary node is new added Epsilon
7. add 6th node and ensure, that new node is catchuped
"""
for __ in range(4):
ensure_view_change(looper,
txnPoolNodeSet,
custom_timeout=tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper,
txnPoolNodeSet,
customTimeout=tconf.VIEW_CHANGE_TIMEOUT)
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
for node in txnPoolNodeSet:
looper.run(eventually(catchuped, node, timeout=2 * timeout))
ensure_all_nodes_have_same_data(looper,
txnPoolNodeSet,
custom_timeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
new_epsilon_node = add_new_node(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath,
name='Epsilon')
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
"""
check that pool and domain ledgers for new node are in synced state
"""
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
for node in txnPoolNodeSet:
looper.run(
eventually(check_ledger_state,
node,
DOMAIN_LEDGER_ID,
LedgerState.synced,
retryWait=.5,
timeout=timeout))
looper.run(
eventually(check_ledger_state,
node,
POOL_LEDGER_ID,
LedgerState.synced,
retryWait=.5,
timeout=timeout))
for __ in range(4):
ensure_view_change(looper,
txnPoolNodeSet,
custom_timeout=tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper,
txnPoolNodeSet,
customTimeout=tconf.VIEW_CHANGE_TIMEOUT)
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
for node in txnPoolNodeSet:
looper.run(eventually(catchuped, node, timeout=3 * timeout))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 2)
new_psi_node = add_new_node(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath,
name='Psi')
looper.run(
eventually(check_ledger_state,
new_psi_node,
DOMAIN_LEDGER_ID,
LedgerState.synced,
retryWait=.5,
timeout=5))
looper.run(
eventually(check_ledger_state,
new_psi_node,
POOL_LEDGER_ID,
LedgerState.synced,
retryWait=.5,
timeout=5))
