def checkNodeDataForEquality(node: TestNode,
*otherNodes: Iterable[TestNode]):
# Checks for node's ledgers and state's to be equal
for n in otherNodes:
check_last_ordered_3pc(node, n)
check_seqno_db_equality(node.seqNoDB, n.seqNoDB)
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkStateEquality(node.getState(DOMAIN_LEDGER_ID), n.getState(DOMAIN_LEDGER_ID))
if n.poolLedger:
checkLedgerEquality(node.poolLedger, n.poolLedger)
checkStateEquality(node.getState(POOL_LEDGER_ID), n.getState(POOL_LEDGER_ID))
def checkNodeDataForEquality(node: TestNode,
*otherNodes: Iterable[TestNode],
exclude_from_check=None):
# Checks for node's ledgers and state's to be equal
for n in otherNodes:
if exclude_from_check != 'check_last_ordered_3pc':
check_last_ordered_3pc(node, n)
else:
logger.debug("Excluding check_last_ordered_3pc check")
check_seqno_db_equality(node.seqNoDB, n.seqNoDB)
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkStateEquality(node.getState(DOMAIN_LEDGER_ID),
n.getState(DOMAIN_LEDGER_ID))
if n.poolLedger:
checkLedgerEquality(node.poolLedger, n.poolLedger)
checkStateEquality(node.getState(POOL_LEDGER_ID),
n.getState(POOL_LEDGER_ID))
def chk_ledger_and_state(first_node, second_node, ledger_id):
checkLedgerEquality(first_node.getLedger(ledger_id),
second_node.getLedger(ledger_id))
if not exclude_from_check or 'check_state' not in exclude_from_check:
checkStateEquality(first_node.getState(ledger_id),
second_node.getState(ledger_id))
def checkClientPoolLedgerSameAsNodes(client: TestClient, *nodes: TestNode):
for n in nodes:
checkLedgerEquality(client.ledger, n.poolLedger)
def testOrderingCase2(looper, nodeSet, up, client1, wallet1):
"""
Scenario -> A client sends requests, some nodes delay COMMITs to few
specific nodes such some nodes achieve commit quorum later for those
requests compared to other nodes. But all nodes `ORDER` request in the same
order of ppSeqNos
https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
"""
pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
getNonPrimaryReplicas(nodeSet, instId=0)
assert len(replicas) == 6
rep0 = pr
rep1 = replicas[0]
rep2 = replicas[1]
rep3 = replicas[2]
rep4 = replicas[3]
rep5 = replicas[4]
rep6 = replicas[5]
node0 = rep0.node
node1 = rep1.node
node2 = rep2.node
node3 = rep3.node
node4 = rep4.node
node5 = rep5.node
node6 = rep6.node
ppSeqsToDelay = 5
commitDelay = 3 # delay each COMMIT by this number of seconds
delayedPpSeqNos = set()
requestCount = 10
def specificCommits(wrappedMsg):
nonlocal node3, node4, node5
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare):
if len(delayedPpSeqNos) < ppSeqsToDelay:
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} be delayed'.format(msg.ppSeqNo))
if isinstance(msg, Commit) and msg.instId == 0 and \
sender in (n.name for n in (node3, node4, node5)) and \
msg.ppSeqNo in delayedPpSeqNos:
return commitDelay
for node in (node1, node2):
logger.debug('{} would be delaying commits'.format(node))
node.nodeIbStasher.delay(specificCommits)
requests = sendRandomRequests(wallet1, client1, requestCount)
waitForSufficientRepliesForRequests(looper, client1, requests=requests)
def ensureSlowNodesHaveAllTxns():
nonlocal node1, node2
for node in node1, node2:
assert len(node.domainLedger) == requestCount
timeout = waits.expectedPoolGetReadyTimeout(len(nodeSet))
looper.run(
eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=timeout))
checkAllLedgersEqual(
(n.domainLedger for n in (node0, node3, node4, node5, node6)))
for node in (node1, node2):
for n in nodeSet:
if n != node:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def testOrderingCase1(looper, nodeSet, up, client1, wallet1):
"""
Scenario -> A client sends requests, some nodes delay COMMITs to few
specific nodes such some nodes achieve commit quorum later for those
requests compared to other nodes. But all nodes `ORDER` request in the same
order of ppSeqNos
https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
"""
pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
getNonPrimaryReplicas(nodeSet, instId=0)
assert len(replicas) == 6
rep0 = pr
rep1 = replicas[0]
rep2 = replicas[1]
rep3 = replicas[2]
rep4 = replicas[3]
rep5 = replicas[4]
rep6 = replicas[5]
node0 = rep0.node
node1 = rep1.node
node2 = rep2.node
node3 = rep3.node
node4 = rep4.node
node5 = rep5.node
node6 = rep6.node
requests = sendRandomRequests(wallet1, client1, 15)
ppSeqsToDelay = 5
delayedPpSeqNos = set()
def specificCommits(wrappedMsg):
nonlocal node3, node4, node5
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare):
if len(delayedPpSeqNos) < ppSeqsToDelay:
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} corresponding to request id {} would '
'be delayed'.format(msg.ppSeqNo, msg.reqId))
if isinstance(msg, Commit) and msg.instId == 0 and \
sender in (n.name for n in (node3, node4, node5)) and \
msg.ppSeqNo in delayedPpSeqNos:
return 3
for node in (node1, node2):
logger.debug('{} would be delaying commits'.format(node))
node.nodeIbStasher.delay(specificCommits)
checkSufficientRepliesForRequests(looper, client1, requests)
def ensureSlowNodesHaveAllTxns():
nonlocal node1, node2
for node in node1, node2:
assert len(node.domainLedger) == 15
looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15))
checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4,
node5, node6)))
for node in (node1, node2):
for n in nodeSet:
if n != node:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def checkNodeLedgersForEquality(node: TestNode,
*otherNodes: Iterable[TestNode]):
for n in otherNodes:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkLedgerEquality(node.poolLedger, n.poolLedger)
def chk_ledger_and_state(first_node, second_node, ledger_id):
checkLedgerEquality(first_node.getLedger(ledger_id),
second_node.getLedger(ledger_id))
checkStateEquality(first_node.getState(ledger_id),
second_node.getState(ledger_id))
def chk_ledger_and_state(first_node, second_node, ledger_id):
checkLedgerEquality(first_node.getLedger(ledger_id),
second_node.getLedger(ledger_id))
if not exclude_from_check or 'check_state' not in exclude_from_check:
checkStateEquality(first_node.getState(ledger_id),
second_node.getState(ledger_id))
def checkClientPoolLedgerSameAsNodes(client: TestClient,
*nodes: Iterable[TestNode]):
for n in nodes:
checkLedgerEquality(client.ledger, n.poolLedger)
def checkNodeLedgersForEquality(node: TestNode,
*otherNodes: Iterable[TestNode]):
for n in otherNodes:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkLedgerEquality(node.poolLedger, n.poolLedger)