def testTreeRootsCorrectAfterEachBatch(tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Check if both state root and txn tree root are correct and same on each
node after each batch
:return:
"""
# Send 1 batch
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
# Send 2 batches
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
def test_catchup_during_3pc(tconf, looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle):
reqs = sdk_signed_random_requests(looper, sdk_wallet_client,
tconf.Max3PCBatchSize)
non_primary_replica = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0]
# Simulate catch-up (add txns to ledger):
# add txns corresponding to the requests after we got enough COMMITs to
# order, but before ordering.
add_txns_to_ledger_before_order(
non_primary_replica,
[json.loads(req) for req in reqs[:tconf.Max3PCBatchSize]])
sdk_send_and_check(reqs, looper, txnPoolNodeSet, sdk_pool_handle)
checkNodesHaveSameRoots(txnPoolNodeSet)
def testTreeRootsCorrectAfterEachBatch(tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Check if both state root and txn tree root are correct and same on each
node after each batch
:return:
"""
# Send 1 batch
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
# Send 2 batches
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
def send_and_check(signed_reqs, looper, txnPoolNodeSet, pool_h, timeout=None):
if not timeout:
timeout_per_request = waits.expectedTransactionExecutionTime(
len(txnPoolNodeSet))
# here we try to take into account what timeout for execution
# N request - total_timeout should be in
# timeout_per_request < total_timeout < timeout_per_request * N
# we cannot just take (timeout_per_request * N) because it is so huge.
# (for timeout_per_request=5 and N=10, total_timeout=50sec)
# lets start with some simple formula:
timeout = (1 + len(signed_reqs) / 10) * timeout_per_request
results = sdk_send_signed_requests(pool_h, signed_reqs)
sdk_get_replies(looper, results, timeout=timeout)
checkNodesHaveSameRoots(txnPoolNodeSet)
def testTreeRootsCorrectAfterEachBatch(tconf, looper, txnPoolNodeSet, client,
wallet1):
"""
Check if both state root and txn tree root are correct and same on each
node after each batch
:return:
"""
# Send 1 batch
reqs = sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize)
waitForSufficientRepliesForRequests(looper, client, requests=reqs)
checkNodesHaveSameRoots(txnPoolNodeSet)
# Send 2 batches
reqs = sendRandomRequests(wallet1, client, 2 * tconf.Max3PCBatchSize)
waitForSufficientRepliesForRequests(looper, client, requests=reqs)
checkNodesHaveSameRoots(txnPoolNodeSet)
def testPrePrepareProcessedInOrder(tconf, looper, txnPoolNodeSet, wallet1,
client):
"""
A non-primary receives PRE-PREPARE out of order, it receives with ppSeqNo 2
earlier than it receives the one with ppSeqNo 1 but it stashes the one
with ppSeqNo 2 and only unstashes it for processing once it has
processed PRE-PREPARE with ppSeqNo 1
:return:
"""
pr, otherR = getPrimaryReplica(txnPoolNodeSet, instId=0), \
getNonPrimaryReplicas(txnPoolNodeSet, instId=0)
otherNodes = [r.node for r in otherR]
ppsToDelay = 2
ppDelay = 3
delayeds = 0
expectedDelayeds = (len(txnPoolNodeSet) - 1) * ppsToDelay
delayedPpSeqNos = set()
def specificPrePrepares(wrappedMsg):
nonlocal delayeds
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare) and delayeds < expectedDelayeds:
delayeds += 1
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} would be delayed'.format(msg.ppSeqNo))
return ppDelay
for node in otherNodes:
logger.debug(
'{} would be delaying reception of some pre-prepares'.format(node))
node.nodeIbStasher.delay(specificPrePrepares)
reqs = sendRandomRequests(wallet1, client,
(ppsToDelay + 1) * tconf.Max3PCBatchSize)
waitForSufficientRepliesForRequests(looper, client, requests=reqs)
checkNodesHaveSameRoots(txnPoolNodeSet)
for r in otherR:
seqNos = [a['pp'].ppSeqNo for a in getAllArgs(r, r.addToPrePrepares)]
seqNos.reverse()
assert sorted(seqNos) == seqNos
def testPrePrepareProcessedInOrder(perf_chk_patched, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
A non-primary receives PRE-PREPARE out of order, it receives with ppSeqNo 2
earlier than it receives the one with ppSeqNo 1 but it stashes the one
with ppSeqNo 2 and only unstashes it for processing once it has
processed PRE-PREPARE with ppSeqNo 1
:return:
"""
tconf = perf_chk_patched
pr, otherR = getPrimaryReplica(txnPoolNodeSet, instId=0), \
getNonPrimaryReplicas(txnPoolNodeSet, instId=0)
otherNodes = [r.node for r in otherR]
ppsToDelay = 2
delayeds = 0
expectedDelayeds = (len(txnPoolNodeSet) - 1) * ppsToDelay
delayedPpSeqNos = set()
def specificPrePrepares(wrappedMsg):
nonlocal delayeds
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare) and delayeds < expectedDelayeds:
delayeds += 1
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} would be delayed'.format(msg.ppSeqNo))
return pp_delay
for node in otherNodes:
logger.debug('{} would be delaying reception of some pre-prepares'.
format(node))
node.nodeIbStasher.delay(specificPrePrepares)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
(ppsToDelay + 1) * tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
for r in otherR:
seqNos = [a['pp'].ppSeqNo for a in getAllArgs(r, r._ordering_service._add_to_pre_prepares)]
seqNos.reverse()
assert sorted(seqNos) == seqNos
def testPrePrepareProcessedInOrder(perf_chk_patched, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
A non-primary receives PRE-PREPARE out of order, it receives with ppSeqNo 2
earlier than it receives the one with ppSeqNo 1 but it stashes the one
with ppSeqNo 2 and only unstashes it for processing once it has
processed PRE-PREPARE with ppSeqNo 1
:return:
"""
tconf = perf_chk_patched
pr, otherR = getPrimaryReplica(txnPoolNodeSet, instId=0), \
getNonPrimaryReplicas(txnPoolNodeSet, instId=0)
otherNodes = [r.node for r in otherR]
ppsToDelay = 2
delayeds = 0
expectedDelayeds = (len(txnPoolNodeSet) - 1) * ppsToDelay
delayedPpSeqNos = set()
def specificPrePrepares(wrappedMsg):
nonlocal delayeds
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare) and delayeds < expectedDelayeds:
delayeds += 1
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} would be delayed'.format(msg.ppSeqNo))
return pp_delay
for node in otherNodes:
logger.debug('{} would be delaying reception of some pre-prepares'.
format(node))
node.nodeIbStasher.delay(specificPrePrepares)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
(ppsToDelay + 1) * tconf.Max3PCBatchSize)
checkNodesHaveSameRoots(txnPoolNodeSet)
for r in otherR:
seqNos = [a['pp'].ppSeqNo for a in getAllArgs(r, r.addToPrePrepares)]
seqNos.reverse()
assert sorted(seqNos) == seqNos
def test_unordered_state_reverted_before_catchup(tconf, looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle):
"""
Check that unordered state is reverted before starting catchup:
- save the initial state on a node
- slow down processing of COMMITs
- send requests
- wait until other nodes come to consensus
- call start of catch-up
- check that the state of the slow node is reverted and equal to the initial one.
"""
# CONFIG
ledger_id = DOMAIN_LEDGER_ID
non_primary_node = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0].node
non_primary_ledger = non_primary_node.getLedger(ledger_id)
non_primary_state = non_primary_node.getState(ledger_id)
# send reqs and make sure we are at the same state
reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 10)
sdk_send_and_check(reqs, looper, txnPoolNodeSet, sdk_pool_handle)
checkNodesHaveSameRoots(txnPoolNodeSet)
# the state of the node before
committed_ledger_before = non_primary_ledger.tree.root_hash
uncommitted_ledger_before = non_primary_ledger.uncommittedRootHash
committed_state_before = non_primary_state.committedHeadHash
uncommitted_state_before = non_primary_state.headHash
# EXECUTE
# Delay commit requests on the node
non_primary_node.nodeIbStasher.delay(cDelay())
# Delay Consistency proofs to not finish catchup
non_primary_node.nodeIbStasher.delay(cpDelay())
# send requests
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
tconf.Max3PCBatchSize)
sdk_get_replies(looper, reqs, timeout=40)
committed_ledger_during_3pc = non_primary_node.getLedger(
ledger_id).tree.root_hash
uncommitted_ledger_during_3pc = non_primary_node.getLedger(
ledger_id).uncommittedRootHash
committed_state_during_3pc = non_primary_node.getState(
ledger_id).committedHeadHash
uncommitted_state_during_3pc = non_primary_node.getState(
ledger_id).headHash
# start catchup
non_primary_node.start_catchup()
committed_ledger_reverted = non_primary_ledger.tree.root_hash
uncommitted_ledger_reverted = non_primary_ledger.uncommittedRootHash
committed_state_reverted = non_primary_state.committedHeadHash
uncommitted_state_reverted = non_primary_state.headHash
# CHECK
# check that initial uncommitted state differs from the state during 3PC
# but committed does not
assert committed_ledger_before == committed_ledger_during_3pc
assert uncommitted_ledger_before != uncommitted_ledger_during_3pc
assert committed_state_before == committed_state_during_3pc
assert uncommitted_state_before != uncommitted_state_during_3pc
assert committed_ledger_before == committed_ledger_reverted
assert uncommitted_ledger_before == uncommitted_ledger_reverted
assert committed_state_before == committed_state_reverted
assert uncommitted_state_before == uncommitted_state_reverted
def test_unordered_state_reverted_before_catchup(
tconf, looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle):
"""
Check that unordered state is reverted before starting catchup:
- save the initial state on a node
- slow down processing of COMMITs
- send requests
- wait until other nodes come to consensus
- call start of catch-up
- check that the state of the slow node is reverted and equal to the initial one.
"""
# CONFIG
ledger_id = DOMAIN_LEDGER_ID
non_primary_node = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0].node
non_primary_ledger = non_primary_node.getLedger(ledger_id)
non_primary_state = non_primary_node.getState(ledger_id)
# send reqs and make sure we are at the same state
reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 10)
sdk_send_and_check(reqs, looper, txnPoolNodeSet, sdk_pool_handle)
checkNodesHaveSameRoots(txnPoolNodeSet)
# the state of the node before
committed_ledger_before = non_primary_ledger.tree.root_hash
uncommitted_ledger_before = non_primary_ledger.uncommittedRootHash
committed_state_before = non_primary_state.committedHeadHash
uncommitted_state_before = non_primary_state.headHash
# EXECUTE
# Delay commit requests on the node
delay_c = 60
non_primary_node.nodeIbStasher.delay(cDelay(delay_c))
# send requests
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, tconf.Max3PCBatchSize)
sdk_get_replies(looper, reqs, timeout=40)
committed_ledger_during_3pc = non_primary_node.getLedger(
ledger_id).tree.root_hash
uncommitted_ledger_during_3pc = non_primary_node.getLedger(
ledger_id).uncommittedRootHash
committed_state_during_3pc = non_primary_node.getState(
ledger_id).committedHeadHash
uncommitted_state_during_3pc = non_primary_node.getState(
ledger_id).headHash
# start catchup
non_primary_node.ledgerManager.preCatchupClbk(ledger_id)
committed_ledger_reverted = non_primary_ledger.tree.root_hash
uncommitted_ledger_reverted = non_primary_ledger.uncommittedRootHash
committed_state_reverted = non_primary_state.committedHeadHash
uncommitted_state_reverted = non_primary_state.headHash
# CHECK
# check that initial uncommitted state differs from the state during 3PC
# but committed does not
assert committed_ledger_before == committed_ledger_during_3pc
assert uncommitted_ledger_before != uncommitted_ledger_during_3pc
assert committed_state_before == committed_state_during_3pc
assert uncommitted_state_before != uncommitted_state_during_3pc
assert committed_ledger_before == committed_ledger_reverted
assert uncommitted_ledger_before == uncommitted_ledger_reverted
assert committed_state_before == committed_state_reverted
assert uncommitted_state_before == uncommitted_state_reverted