def test_handle_delayed_preprepares(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
Make a node send PREPREPARE again after the slow node has ordered
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = \
split_nodes(txnPoolNodeSet)
# This node will send PRE-PREPARE again
orig_method = primary_node.handlers[PREPREPARE].serve
last_pp = None
def patched_method(self, msg):
nonlocal last_pp
last_pp = orig_method(msg)
return last_pp
primary_node.handlers[PREPREPARE].serve = types.MethodType(patched_method,
primary_node.handlers[
PREPREPARE])
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
slow_master_replica = slow_node.master_replica
count_pr_req = get_count(slow_master_replica,
slow_master_replica.process_requested_pre_prepare)
count_pr_tpc = get_count(slow_master_replica,
slow_master_replica.processThreePhaseMsg)
primary_node.sendToNodes(MessageRep(**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {
f.INST_ID.nm: last_pp.instId,
f.VIEW_NO.nm: last_pp.viewNo,
f.PP_SEQ_NO.nm: last_pp.ppSeqNo
},
f.MSG.nm: last_pp
}), names=[slow_node.name, ])
def chk():
# `process_requested_pre_prepare` is called but
# `processThreePhaseMsg` is not called
assert get_count(
slow_master_replica,
slow_master_replica.process_requested_pre_prepare) > count_pr_req
assert get_count(
slow_master_replica,
slow_master_replica.processThreePhaseMsg) == count_pr_tpc
looper.run(eventually(chk, retryWait=1))
def test_no_preprepare_requested(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay(20))
slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ]))
old_count_resp = count_requested_preprepare_resp(slow_node)
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 4, 2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle, teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay())
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(msg_rep_delay(1000, [
PROPAGATE,
]))
old_count_resp = count_requested_preprepare_resp(slow_node)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=4,
num_batches=2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_no_preprepare_requested(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay(20))
slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ]))
old_count_resp = count_requested_preprepare_resp(slow_node)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=4,
num_batches=2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_handle_delayed_preprepares(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle, teardown, monkeypatch):
"""
Make a node send PREPREPARE again after the slow node has ordered
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = \
split_nodes(txnPoolNodeSet)
# This node will send PRE-PREPARE again
orig_method = primary_node.master_replica._message_req_service.handlers[
PREPREPARE].process_message_req
handler = primary_node.master_replica._message_req_service.handlers[
PREPREPARE]
last_pp = None
def patched_method(self, msg):
nonlocal last_pp
last_pp = orig_method(msg)
return last_pp
handler.process_message_req = types.MethodType(patched_method, handler)
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
slow_master_replica = slow_node.master_replica
count_pr_req = get_count(
slow_master_replica._message_req_service,
slow_master_replica._message_req_service.process_message_rep)
primary_node.sendToNodes(MessageRep(
**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {
f.INST_ID.nm: last_pp.instId,
f.VIEW_NO.nm: last_pp.viewNo,
f.PP_SEQ_NO.nm: last_pp.ppSeqNo
},
f.MSG.nm: last_pp
}),
names=[
slow_node.name,
])
def chk():
assert get_count(
slow_master_replica._message_req_service, slow_master_replica.
_message_req_service.process_message_rep) > count_pr_req
assert slow_master_replica._ordering_service.spylog.getLast(
"_validate").result[0] == -1
looper.run(eventually(chk, retryWait=1))
def test_node_request_preprepare(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, teardown):
"""
Node requests PRE-PREPARE only once after getting PREPAREs.
"""
slow_node, other_nodes, primary_node, \
other_primary_nodes = split_nodes(txnPoolNodeSet)
# Drop PrePrepares and Prepares
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
slow_node.nodeIbStasher.delay(pDelay(300, 0))
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 10, 5)
slow_node.nodeIbStasher.drop_delayeds()
slow_node.nodeIbStasher.resetDelays()
old_count_req = count_requested_preprepare_req(slow_node)
old_count_resp = count_requested_preprepare_resp(slow_node)
def chk(increase=True):
# Method is called
assert count_requested_preprepare_req(slow_node) > old_count_req
# Requesting Preprepare
assert count_requested_preprepare_resp(
slow_node) - old_count_resp == (1 if increase else 0)
for pp in primary_node.master_replica.sentPrePrepares.values():
for rep in [n.master_replica for n in other_primary_nodes]:
prepare = Prepare(rep.instId,
pp.viewNo,
pp.ppSeqNo,
pp.ppTime,
pp.digest,
pp.stateRootHash,
pp.txnRootHash
)
rep.send(prepare)
looper.run(eventually(chk, True, retryWait=1))
old_count_resp = count_requested_preprepare_resp(slow_node)
prepare = Prepare(rep.instId,
pp.viewNo,
pp.ppSeqNo,
pp.ppTime,
pp.digest,
pp.stateRootHash,
pp.txnRootHash
)
rep.send(prepare)
looper.run(eventually(chk, False, retryWait=1))
old_count_req = count_requested_preprepare_req(slow_node)
old_count_resp = count_requested_preprepare_resp(slow_node)
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
A node has bad network with primary and thus loses PRE-PREPARE,
it requests PRE-PREPARE from primary once it has sufficient PREPAREs
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = split_nodes(
txnPoolNodeSet)
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
old_count_pp = get_count(
slow_node.master_replica,
slow_node.master_replica._ordering_service.process_preprepare)
old_count_mrq = {
n.name: get_count(n, n.process_message_req)
for n in other_nodes
}
old_count_mrp = get_count(slow_node, slow_node.process_message_rep)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=15,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
assert not slow_node.master_replica.requested_pre_prepares
# `slow_node` processed PRE-PREPARE
# assert get_count(slow_node.master_replica,
# slow_node.master_replica._ordering_service.process_preprepare) > old_count_pp
# `slow_node` did receive `MessageRep`
assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp
# Primary node should received `MessageReq` and other nodes shouldn't
recv_reqs = set()
for n in other_non_primary_nodes:
if get_count(n, n.process_message_req) > old_count_mrq[n.name]:
recv_reqs.add(n.name)
assert get_count(primary_node, primary_node.process_message_req) > \
old_count_mrq[primary_node.name]
assert len(recv_reqs) == 0
# All nodes including the `slow_node` ordered the same requests
assert check_if_all_equal_in_list(
[n.master_replica._ordering_service.ordered for n in txnPoolNodeSet])
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
A node has bad network with primary and thus loses PRE-PREPARE,
it requests PRE-PREPARE from primary once it has sufficient PREPAREs
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = split_nodes(
txnPoolNodeSet)
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
old_count_pp = get_count(slow_node.master_replica,
slow_node.master_replica.processPrePrepare)
old_count_mrq = {n.name: get_count(n, n.process_message_req)
for n in other_nodes}
old_count_mrp = get_count(slow_node, slow_node.process_message_rep)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=15,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
assert not slow_node.master_replica.requested_pre_prepares
# `slow_node` processed PRE-PREPARE
assert get_count(slow_node.master_replica,
slow_node.master_replica.processPrePrepare) > old_count_pp
# `slow_node` did receive `MessageRep`
assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp
# Primary node should received `MessageReq` and other nodes shouldn't
recv_reqs = set()
for n in other_non_primary_nodes:
if get_count(n, n.process_message_req) > old_count_mrq[n.name]:
recv_reqs.add(n.name)
assert get_count(primary_node, primary_node.process_message_req) > \
old_count_mrq[primary_node.name]
assert len(recv_reqs) == 0
# All nodes including the `slow_node` ordered the same requests
assert check_if_all_equal_in_list([n.master_replica.ordered
for n in txnPoolNodeSet])
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, teardown):
"""
A node has bad network with primary and thus loses PRE-PREPARE,
it requests PRE-PREPARE from non-primaries once it has sufficient PREPAREs
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
old_count_pp = get_count(slow_node.master_replica,
slow_node.master_replica.processPrePrepare)
old_count_mrq = {
n.name: get_count(n, n.process_message_req)
for n in other_nodes
}
old_count_mrp = get_count(slow_node, slow_node.process_message_rep)
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 15, 5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
assert not slow_node.master_replica.requested_pre_prepares
# `slow_node` processed PRE-PREPARE
assert get_count(slow_node.master_replica,
slow_node.master_replica.processPrePrepare) > old_count_pp
# `slow_node` did receive `MessageRep`
assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp
# More than `f` nodes received `MessageReq`
recv_reqs = set()
for n in other_nodes:
if get_count(n, n.process_message_req) > old_count_mrq[n.name]:
recv_reqs.add(n.name)
assert len(recv_reqs) > slow_node.f
# All nodes including the `slow_node` ordered the same requests
assert check_if_all_equal_in_list(
[n.master_replica.ordered for n in txnPoolNodeSet])
def test_node_request_preprepare(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
Node requests PRE-PREPARE only once after getting PREPAREs.
"""
slow_node, other_nodes, primary_node, \
other_primary_nodes = split_nodes(txnPoolNodeSet)
# Drop PrePrepares and Prepares
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
slow_node.nodeIbStasher.delay(pDelay(300, 0))
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
slow_node.nodeIbStasher.drop_delayeds()
slow_node.nodeIbStasher.resetDelays()
old_count_req = count_requested_preprepare_req(slow_node)
old_count_resp = count_requested_preprepare_resp(slow_node)
def chk(increase=True):
# Method is called
assert count_requested_preprepare_req(slow_node) > old_count_req
# Requesting Preprepare
assert count_requested_preprepare_resp(
slow_node) - old_count_resp == (1 if increase else 0)
for pp in primary_node.master_replica.sentPrePrepares.values():
for rep in [n.master_replica for n in other_primary_nodes]:
prepare = Prepare(rep.instId,
pp.viewNo,
pp.ppSeqNo,
pp.ppTime,
pp.digest,
pp.stateRootHash,
pp.txnRootHash
)
rep.send(prepare)
looper.run(eventually(chk, True, retryWait=1))
old_count_resp = count_requested_preprepare_resp(slow_node)
prepare = Prepare(rep.instId,
pp.viewNo,
pp.ppSeqNo,
pp.ppTime,
pp.digest,
pp.stateRootHash,
pp.txnRootHash
)
rep.send(prepare)
looper.run(eventually(chk, False, retryWait=1))
old_count_req = count_requested_preprepare_req(slow_node)
old_count_resp = count_requested_preprepare_resp(slow_node)
