def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logger.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1, wallet1 = setupClient(looper,
nodeSet,
tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(
eventually(checkRequestReturnedToNode,
node,
wallet1.defaultId,
req.reqId,
req.digest,
instNo,
retryWait=1,
timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logging.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
client1.defaultIdentifier, req.reqId,
req.digest,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1,
timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=20))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir,
pluginVerPath):
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir)
req = submitOp(wallet1, client1, {
TXN_TYPE: "dummy",
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid "
"transaction type, must be "
"one of {}".format(validTypes))
}
coros1 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros2 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: "some string"
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros3 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: -3
}
})
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
coros4 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
timeout = waits.expectedReqAckQuorumTime()
looper.run(
eventuallyAll(*(coros1 + coros2 + coros3 + coros4),
totalTimeout=timeout))
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
waitForSufficientRepliesForRequests(looper,
client1,
requests=[req],
fVal=1)
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 1
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=timeout))
def assert_msg_count(typ, count):
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
PrePrepare,
1,
retryWait=1,
timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
Prepare,
2,
retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=delayD)) # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir,
pluginVerPath):
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir)
req = submitOp(wallet1, client1, {
TXN_TYPE: "dummy",
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}})
validTypes = ', '.join(plugin.validTxnTypes)
update = {'reason': makeReason(commonError, "dummy is not a valid "
"transaction type, must be "
"one of {}".format(validTypes))}
coros1 = [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
})
update = {
'reason': makeReason(commonError,
"{} attribute is missing or not in proper format"
.format(DATA))}
coros2 = [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: "some string"})
update = {
'reason': makeReason(commonError,
"{} attribute is missing or not in proper format"
.format(DATA))}
coros3 = [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: -3
}})
update = {
'reason': makeReason(commonError, "{} must be present and should be "
"a number greater than 0"
.format(AMOUNT))}
coros4 = [partial(checkReqNack, client1, node, req.identifier, req.reqId,
update) for node in nodeSet]
looper.run(eventuallyAll(*(coros1+coros2+coros3+coros4), totalTimeout=5))
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}})
looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox,
req.reqId, 1,
retryWait=1, timeout=5))
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 1
