def testZStackSendRecvHugeDataUnderLimit(set_info_log_level, tdir, looper, tconf):
names = ['Alpha', 'Beta']
genKeys(tdir, names)
# we use json serializer now, so final form will be {'k':'vvv...vvv'}
# so here we try to prepare exactly tconf.MSG_LEN_LIMIT bytes after serialization
msg = {'k': 'v' * (tconf.MSG_LEN_LIMIT - len("{'k':''}"))}
betaHandler = [False]
def recvHandlerAlpha(wrpMsg):
pass
def recvHandlerBeta(wrpMsg):
rmsg, frm = wrpMsg
betaHandler[0] = True
assert frm == 'Alpha'
assert rmsg == msg
alpha = ZStack(names[0], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerAlpha, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=None)
beta = ZStack(names[1], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerBeta, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=None)
assert len(alpha.serializeMsg(msg)) == tconf.MSG_LEN_LIMIT
prepStacks(looper, *(alpha, beta), connect=True, useKeys=True)
stat = alpha.send(msg, beta.name)
assert stat[0] is True
looper.runFor(5)
assert betaHandler[0] is True
def testZStackSendRecvHugeDataUnderLimit(set_info_log_level, tdir, looper, tconf):
names = ['Alpha', 'Beta']
genKeys(tdir, names)
# we use json serializer now, so final form will be {'k':'vvv...vvv'}
# so here we try to prepare exactly tconf.MSG_LEN_LIMIT bytes after serialization
msg = {'k': 'v' * (tconf.MSG_LEN_LIMIT - len("{'k':''}"))}
betaHandler = [False]
def recvHandlerAlpha(wrpMsg):
pass
def recvHandlerBeta(wrpMsg):
rmsg, frm = wrpMsg
betaHandler[0] = True
assert frm == 'Alpha'
assert rmsg == msg
alpha = ZStack(names[0], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerAlpha, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=None)
beta = ZStack(names[1], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerBeta, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=None)
assert len(alpha.serializeMsg(msg)) == tconf.MSG_LEN_LIMIT
prepStacks(looper, *(alpha, beta), connect=True, useKeys=True)
stat = alpha.send(msg, beta.name)
assert stat[0] is True
looper.runFor(5)
assert betaHandler[0] is True
def testZStackRecvHugeDataOverLimit(set_info_log_level, tdir, looper, tconf):
names = ['Alpha', 'Beta']
genKeys(tdir, names)
# we use json serializer now, so final form will be {'k':'vvv...vvv'}
# so here we try to prepare exactly tconf.MSG_LEN_LIMIT + 1 bytes after serialization
msg = {'k': 'v' * (tconf.MSG_LEN_LIMIT - len("{'k':''}") + 1)}
betaHandlers = [False, False]
def recvHandlerAlpha(wrpMsg):
pass
def recvHandlerBeta(wrpMsg):
rmsg, frm = wrpMsg
betaHandlers[0] = True
assert frm is not None
assert rmsg is not None
def rejectHandlerBeta(reason, frm):
betaHandlers[1] = True
assert 'exceeded allowed limit of {}'.format(
tconf.MSG_LEN_LIMIT) in reason
assert frm == 'Alpha'
alpha = ZStack(names[0],
ha=genHa(),
basedirpath=tdir,
msgHandler=recvHandlerAlpha,
restricted=True,
config=adict(**tconf.__dict__),
msgRejectHandler=None)
beta = ZStack(names[1],
ha=genHa(),
basedirpath=tdir,
msgHandler=recvHandlerBeta,
restricted=True,
config=adict(**tconf.__dict__),
msgRejectHandler=rejectHandlerBeta)
bytemsg = alpha.serializeMsg(msg)
assert len(bytemsg) == (tconf.MSG_LEN_LIMIT + 1)
prepStacks(looper, *(alpha, beta), connect=True, useKeys=True)
stat = alpha._remotes['Beta'].socket.send(bytemsg, copy=False, track=True)
assert stat
looper.runFor(5)
assert betaHandlers[0] is False
assert betaHandlers[1] is True
def nodeRegsForCLI(nodeNames):
nodeNames = ['Alpha', 'Beta', 'Gamma', 'Delta']
has = [genHa(2) for _ in nodeNames]
nodeNamesC = [n + 'C' for n in nodeNames]
nodeReg = OrderedDict((n, has[i][0]) for i, n in enumerate(nodeNames))
cliNodeReg = OrderedDict((n, has[i][1]) for i, n in enumerate(nodeNamesC))
return adict(nodeReg=nodeReg, cliNodeReg=cliNodeReg)
def setup(txnPoolNodeSet):
primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
getNonPrimaryReplicas(txnPoolNodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(
faultyRep.node,
partial(sendDuplicate3PhaseMsg, msgType=Commit, count=3, instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple COMMIT messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in txnPoolNodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_CM_SENT.code)
# If the request is ordered then COMMIT will be rejected much earlier
for r in [primaryRep, *nonPrimaryReps]:
def do_nothing(self, commit):
pass
r._ordering_service._do_order = types.MethodType(do_nothing, r)
return adict(primaryRep=primaryRep,
nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def setup(looper, tconf, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 5)
P = getPrimaryReplica(txnPoolNodeSet)
# set LAMBDA smaller than the production config to make the test faster
testLambda = 30
delay_by = testLambda + 5
for node in txnPoolNodeSet:
# Make `Delta` small enough so throughput check passes.
node.monitor.Delta = .001
node.monitor.Lambda = testLambda
for r in node.replicas:
r.config.ACCEPTABLE_DEVIATION_PREPREPARE_SECS += delay_by
slowed_request = False
# make P (primary replica on master) faulty, i.e., slow to send
# PRE-PREPARE for a specific client request only
def specificPrePrepare(msg):
nonlocal slowed_request
if isinstance(msg, PrePrepare) and slowed_request is False:
slowed_request = True
return delay_by # just more that LAMBDA
P.outBoxTestStasher.delay(specificPrePrepare)
# TODO select or create a timeout for this case in 'waits'
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 5,
customTimeoutPerReq=tconf.TestRunningTimeLimitSec)
return adict(nodes=txnPoolNodeSet)
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in G, Z:
makeNodeFaulty(node, changesRequest)
return adict(faulties=(G, Z))
def setup(startedNodes):
# Making nodes faulty such that no primary is chosen
G = startedNodes.Gamma
# Delaying nomination to avoid becoming primary
# G.delaySelfNomination(10)
makeNodeFaulty(G, partial(delaysPrePrepareProcessing, delay=60))
return adict(faulty=G)
def setup(looper, tconf, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 5)
P = getPrimaryReplica(txnPoolNodeSet)
# set LAMBDA smaller than the production config to make the test faster
testLambda = 10
delay_by = 2 * testLambda
old_view_nos = set([n.viewNo for n in txnPoolNodeSet])
assert len(old_view_nos) == 1
old_view_no = old_view_nos.pop()
for node in txnPoolNodeSet:
# Make `Delta` small enough so throughput check passes.
node.monitor.Delta = .001
node.monitor.Lambda = testLambda
for r in node.replicas.values():
r.config.ACCEPTABLE_DEVIATION_PREPREPARE_SECS += delay_by
# make P (primary replica on master) faulty, i.e., slow to send
# PRE-PREPARE the next
def specificPrePrepare(msg):
if isinstance(msg, PrePrepare):
return delay_by # just more that LAMBDA
P.outBoxTestStasher.delay(specificPrePrepare)
# TODO select or create a timeout for this case in 'waits'
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 5,
customTimeoutPerReq=tconf.TestRunningTimeLimitSec)
return adict(nodes=txnPoolNodeSet, old_view_no=old_view_no)
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
G = txnPoolNodeSet[-1]
# Delaying nomination to avoid becoming primary
# G.delaySelfNomination(10)
makeNodeFaulty(G, partial(delaysPrePrepareProcessing, delay=60))
return adict(faulty=G)
def gen_defs(cls, ips, node_count, node_names, node_ports, client_ports):
"""
Generates some default steward and node definitions for tests
:param ips: array of ip addresses
:param node_count: number of stewards/nodes
:return: duple of steward and node definitions
"""
steward_defs = []
node_defs = []
for i in range(1, node_count + 1):
d = adict()
d.name = "Steward" + str(i)
d.sigseed = cls.get_signing_seed()
s_signer = DidSigner(seed=d.sigseed)
d.nym = s_signer.identifier
d.verkey = s_signer.verkey
steward_defs.append(d)
name = node_names[i - 1]
sigseed = cls.get_signing_seed()
node_defs.append(
NodeDef(name=name,
ip=ips[i - 1],
port=node_ports[i - 1],
client_port=client_ports[i - 1],
idx=i,
sigseed=sigseed,
verkey=Signer(sigseed).verhex,
steward_nym=d.nym))
return steward_defs, node_defs
def setup(looper, tconf, startedNodes, up, wallet1, client1):
# Get the master replica of the master protocol instance
P = getPrimaryReplica(startedNodes)
# Make `Delta` small enough so throughput check passes.
for node in startedNodes:
node.monitor.Delta = .001
# set LAMBDA not so huge like it set in the production config
testLambda = 30
for node in startedNodes:
node.monitor.Lambda = testLambda
slowed_request = False
# make P (primary replica on master) faulty, i.e., slow to send
# PRE-PREPARE for a specific client request only
def specificPrePrepare(msg):
nonlocal slowed_request
if isinstance(msg, PrePrepare) and slowed_request is False:
slowed_request = True
return testLambda + 5 # just more that LAMBDA
P.outBoxTestStasher.delay(specificPrePrepare)
# TODO select or create a timeout for this case in 'waits'
sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
numReqs=5,
customTimeoutPerReq=tconf.TestRunningTimeLimitSec)
return adict(nodes=startedNodes)
def setup(startedNodes):
A = startedNodes.Gamma
B = startedNodes.Zeta
for node in A, B:
makeNodeFaulty(node, partial(delaysPrePrepareProcessing, delay=60))
# node.delaySelfNomination(10)
return adict(faulties=(A, B))
def gen_client_def(cls, idx):
d = adict()
d.name = "Client" + str(idx)
d.sigseed = cls.getSigningSeed(d.name)
d.verkey = Signer(d.sigseed).verhex
d.nym = cls.getNymFromVerkey(d.verkey)
return d
def setup(txnPoolNodeSet):
A = txnPoolNodeSet[-2]
B = txnPoolNodeSet[-1]
for node in A, B:
makeNodeFaulty(node, partial(delaysPrePrepareProcessing, delay=60))
# node.delaySelfNomination(10)
return adict(faulties=(A, B))
def setup(txnPoolNodeSet):
primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
getNonPrimaryReplicas(txnPoolNodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
msgType=Commit, count=3,
instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple COMMIT messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in txnPoolNodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_CM_SENT.code)
# If the request is ordered then COMMIT will be rejected much earlier
for r in [primaryRep, *nonPrimaryReps]:
def do_nothing(self, commit):
pass
r.doOrder = types.MethodType(do_nothing, r)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def setup(txnPoolNodeSet):
E = txnPoolNodeSet[-3]
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in E, G, Z:
makeNodeFaulty(node, changesRequest)
return adict(faulties=(E, G, Z))
def gen_trustee_def(cls, idx):
d = adict()
d.name = 'Trustee' + str(idx)
d.sigseed = cls.getSigningSeed(d.name)
d.verkey = Signer(d.sigseed).verhex
d.nym = cls.getNymFromVerkey(d.verkey)
return d
def setup(nodeSet):
gn = [v for k, v in nodeSet.nodes.items() if k != 'Alpha']
# delay incoming client messages for good nodes by 250 milliseconds
# this gives Alpha a chance to send a propagate message
for n in gn: # type: TestNode
n.clientIbStasher.delay(lambda _: 1)
return adict(goodNodes=gn)
def setup(startedNodes):
# Making nodes faulty such that no primary is chosen
G = startedNodes.Gamma
Z = startedNodes.Zeta
for node in G, Z:
makeNodeFaulty(node, changesRequest)
# node.delaySelfNomination(10)
return adict(faulties=(G, Z))
def setup(txnPoolNodeSet):
pool_without_alpha = list(txnPoolNodeSet)
pool_without_alpha.remove(txnPoolNodeSet[0])
# delay incoming client messages for good nodes by 250 milliseconds
# this gives Alpha a chance to send a propagate message
for n in pool_without_alpha: # type: TestNode
n.clientIbStasher.delay(lambda _: 1)
return adict(goodNodes=pool_without_alpha)
def setup(txnPoolNodeSet):
# A = startedNodes.Alpha
# B = startedNodes.Beta
A, B = nodes_by_rank(txnPoolNodeSet)[-2:]
for node in A, B:
makeNodeFaulty(node, changesRequest,
partial(delaysPrePrepareProcessing, delay=90))
return adict(faulties=(A, B))
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in G, Z:
makeNodeFaulty(node, changesRequest)
# node.delaySelfNomination(10)
return adict(faulties=(G, Z))
def setup(txnPoolNodeSet):
A = txnPoolNodeSet[-2]
B = txnPoolNodeSet[-1]
for node in A, B:
makeNodeFaulty(node,
partial(delaysPrePrepareProcessing, delay=60))
# node.delaySelfNomination(10)
return adict(faulties=(A, B))
def gen_client_def(cls, idx):
d = adict()
d.name = "Client" + str(idx)
d.sigseed = cls.getSigningSeed(d.name)
c_signer = DidSigner(seed=d.sigseed)
d.nym = c_signer.identifier
d.verkey = c_signer.verkey
return d
def gen_trustee_def(cls, idx):
d = adict()
d.name = 'Trustee' + str(idx)
d.sigseed = cls.getSigningSeed(d.name)
t_signer = DidSigner(seed=d.sigseed)
d.nym = t_signer.identifier
d.verkey = t_signer.verkey
return d
def setup(txnPoolNodeSet):
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in G, Z:
makeNodeFaulty(node, partial(delaysPrePrepareProcessing, delay=60))
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(G, Z))
def setup(startedNodes):
G = startedNodes.Gamma
Z = startedNodes.Zeta
for node in G, Z:
makeNodeFaulty(node, partial(delaysPrePrepareProcessing, delay=60))
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(G, Z))
def setup(startedNodes):
E = startedNodes.Eta
Z = startedNodes.Gamma
Z = startedNodes.Zeta
for node in E, Z, Z:
makeNodeFaulty(node, changesRequest)
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(E, Z, Z))
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
A = txnPoolNodeSet[-2]
B = txnPoolNodeSet[-1]
# Delay processing of PRE-PREPARE messages by Alpha and Beta for 90
# seconds since the timeout for checking sufficient commits is 60 seconds
makeNodeFaulty(A, partial(delaysPrePrepareProcessing, delay=90))
makeNodeFaulty(B, partial(delaysPrePrepareProcessing, delay=90))
return adict(faulties=(A, B))
def setup(txnPoolNodeSet):
primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
getNonPrimaryReplicas(txnPoolNodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(txnPoolNodeSet):
E = txnPoolNodeSet[-3]
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in E, G, Z:
makeNodeFaulty(node, changesRequest)
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(E, G, Z))
def setup(startedNodes):
# A = startedNodes.Alpha
# B = startedNodes.Beta
A, B = startedNodes.nodes_by_rank[-2:]
for node in A, B:
makeNodeFaulty(node, changesRequest,
partial(delaysPrePrepareProcessing, delay=90))
# node.delaySelfNomination(10)
return adict(faulties=(A, B))
def stop_nodes(looper, nodeSet):
faulties = nodeSet.nodes_by_rank[-faultyNodes:]
for node in faulties:
for r in node.replicas:
assert not r.isPrimary
disconnect_node_and_ensure_disconnected(
looper, nodeSet, node, stopNode=False)
looper.removeProdable(node)
return adict(faulties=faulties)
def setup(txnPoolNodeSet):
# A = startedNodes.Alpha
# B = startedNodes.Beta
A, B = nodes_by_rank(txnPoolNodeSet)[-2:]
for node in A, B:
makeNodeFaulty(node, changesRequest,
partial(delaysPrePrepareProcessing, delay=90))
# node.delaySelfNomination(10)
return adict(faulties=(A, B))
def setup(txnPoolNodeSet):
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in G, Z:
makeNodeFaulty(node,
partial(delaysPrePrepareProcessing, delay=60))
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(G, Z))
def testZStackRecvHugeDataOverLimit(set_info_log_level, tdir, looper, tconf):
names = ['Alpha', 'Beta']
genKeys(tdir, names)
# we use json serializer now, so final form will be {'k':'vvv...vvv'}
# so here we try to prepare exactly tconf.MSG_LEN_LIMIT + 1 bytes after serialization
msg = {'k': 'v' * (tconf.MSG_LEN_LIMIT - len("{'k':''}") + 1)}
betaHandlers = [False, False]
def recvHandlerAlpha(wrpMsg):
pass
def recvHandlerBeta(wrpMsg):
rmsg, frm = wrpMsg
betaHandlers[0] = True
assert frm is not None
assert rmsg is not None
def rejectHandlerBeta(reason, frm):
betaHandlers[1] = True
assert 'exceeded allowed limit of {}'.format(
tconf.MSG_LEN_LIMIT) in reason
assert frm == 'Alpha'
alpha = ZStack(names[0], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerAlpha, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=None)
beta = ZStack(names[1], ha=genHa(), basedirpath=tdir, msgHandler=recvHandlerBeta, restricted=True,
config=adict(**tconf.__dict__), msgRejectHandler=rejectHandlerBeta)
bytemsg = alpha.serializeMsg(msg)
assert len(bytemsg) == (tconf.MSG_LEN_LIMIT + 1)
prepStacks(looper, *(alpha, beta), connect=True, useKeys=True)
stat = alpha._remotes['Beta'].socket.send(bytemsg, copy=False, track=True)
assert stat
looper.runFor(5)
assert betaHandlers[0] is False
assert betaHandlers[1] is True
def setup(txnPoolNodeSet):
primaryRep, nonPrimaryReps = getPrimaryReplica(txnPoolNodeSet, 0), \
getNonPrimaryReplicas(txnPoolNodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(
primaryRep.node,
partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
E = txnPoolNodeSet[-3]
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in E, G, Z:
makeNodeFaulty(
node, changesRequest,
partial(delaysPrePrepareProcessing, delay=delayPrePrepareSec))
return adict(faulties=(E, G, Z))
def setup(txnPoolNodeSet):
# Making nodes faulty such that no primary is chosen
E = txnPoolNodeSet[-3]
G = txnPoolNodeSet[-2]
Z = txnPoolNodeSet[-1]
for node in E, G, Z:
makeNodeFaulty(node,
changesRequest, partial(delaysPrePrepareProcessing,
delay=delayPrePrepareSec))
# Delaying nomination to avoid becoming primary
# node.delaySelfNomination(10)
return adict(faulties=(E, G, Z))
def step1(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
startedNodes = txnPoolNodeSet
"""
stand up a pool of nodes and send 5 requests to client
"""
# the master instance has a primary replica, call it P
P = getPrimaryReplica(startedNodes)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
# profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5)
return adict(P=P,
nodes=startedNodes,
requests=requests)
def run(self, coro, nodecount=4):
assert self.is_run == False
self.is_run = True
tmpdir = self.tmpdir if self.tmpdir is not None else self.fresh_tdir()
with self.testNodeSetClass(self.config, count=nodecount, tmpdir=tmpdir) as nodeset:
with Looper(nodeset) as looper:
# for n in nodeset:
# n.startKeySharing()
ctx = adict(looper=looper, nodeset=nodeset, tmpdir=tmpdir)
looper.run(checkNodesConnected(nodeset))
ensureElectionsDone(looper=looper,
nodes=nodeset)
looper.run(coro(ctx))
def setup(txnPoolNodeSet):
A = txnPoolNodeSet[-1]
makeNodeFaulty(A,
partial(delaysPrePrepareProcessing, delay=60))
# A.delaySelfNomination(10)
return adict(faulties=A)
