def test_handle_delayed_preprepares(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
Make a node send PREPREPARE again after the slow node has ordered
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = \
split_nodes(txnPoolNodeSet)
# This node will send PRE-PREPARE again
orig_method = primary_node.handlers[PREPREPARE].serve
last_pp = None
def patched_method(self, msg):
nonlocal last_pp
last_pp = orig_method(msg)
return last_pp
primary_node.handlers[PREPREPARE].serve = types.MethodType(patched_method,
primary_node.handlers[
PREPREPARE])
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
slow_master_replica = slow_node.master_replica
count_pr_req = get_count(slow_master_replica,
slow_master_replica.process_requested_pre_prepare)
count_pr_tpc = get_count(slow_master_replica,
slow_master_replica.processThreePhaseMsg)
primary_node.sendToNodes(MessageRep(**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {
f.INST_ID.nm: last_pp.instId,
f.VIEW_NO.nm: last_pp.viewNo,
f.PP_SEQ_NO.nm: last_pp.ppSeqNo
},
f.MSG.nm: last_pp
}), names=[slow_node.name, ])
def chk():
# `process_requested_pre_prepare` is called but
# `processThreePhaseMsg` is not called
assert get_count(
slow_master_replica,
slow_master_replica.process_requested_pre_prepare) > count_pr_req
assert get_count(
slow_master_replica,
slow_master_replica.processThreePhaseMsg) == count_pr_tpc
looper.run(eventually(chk, retryWait=1))
def test_serialization_of_submessages_to_dict():
message = LedgerStatus(1, 10, None, None,
"AwgQhPR9cgRubttBGjRruCRMLhZFBffbejbPipj7WBBm",
CURRENT_PROTOCOL_VERSION)
message_rep = MessageRep(
**{
f.MSG_TYPE.nm: "LEDGER_STATUS",
f.PARAMS.nm: {
"ledger_id": 1,
f.PROTOCOL_VERSION.nm: CURRENT_PROTOCOL_VERSION
},
f.MSG.nm: message,
})
serialized_message = ZStack.serializeMsg(message).decode()
serialized_message_reply = ZStack.serializeMsg(message_rep).decode()
# check that submessage (LedgerStatus) is serialized to the same dict as
# it were a common message
assert serialized_message in serialized_message_reply
# check that de-serialized into the same message
deserialized_message = LedgerStatus(
**ZStack.deserializeMsg(serialized_message))
deserialized_submessage = LedgerStatus(
**ZStack.deserializeMsg(serialized_message_reply)[f.MSG.nm])
assert message == deserialized_message
assert message_rep.msg == deserialized_submessage
assert message == deserialized_submessage
def process_message_req(self, msg: MessageReq, frm):
# Assumes a shared memory architecture. In case of multiprocessing,
# RPC architecture, use deques to communicate the message and node will
# maintain a unique internal message id to correlate responses.
msg_type = msg.msg_type
if msg_type not in self.handlers.keys():
self.discard(msg, "Unknown message type {}".format(msg_type),
self._logger.warning)
return
handler = self.handlers[msg_type]
try:
resp = handler.process_message_req(msg)
except IncorrectMessageForHandlingException as e:
self.discard(e.msg, e.reason, e.log_method)
return
if not resp:
return
with self.metrics.measure_time(MetricsName.SEND_MESSAGE_REP_TIME):
self._network.send(MessageRep(
**{
f.MSG_TYPE.nm: msg_type,
f.PARAMS.nm: msg.params,
f.MSG.nm: resp
}),
dst=[
frm.split(":")[0],
])
def test_handle_delayed_preprepares(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle, teardown, monkeypatch):
"""
Make a node send PREPREPARE again after the slow node has ordered
"""
slow_node, other_nodes, primary_node, other_non_primary_nodes = \
split_nodes(txnPoolNodeSet)
# This node will send PRE-PREPARE again
orig_method = primary_node.master_replica._message_req_service.handlers[
PREPREPARE].process_message_req
handler = primary_node.master_replica._message_req_service.handlers[
PREPREPARE]
last_pp = None
def patched_method(self, msg):
nonlocal last_pp
last_pp = orig_method(msg)
return last_pp
handler.process_message_req = types.MethodType(patched_method, handler)
# Delay PRE-PREPAREs by large amount simulating loss
slow_node.nodeIbStasher.delay(ppDelay(300, 0))
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
waitNodeDataEquality(looper, slow_node, *other_nodes)
slow_master_replica = slow_node.master_replica
count_pr_req = get_count(
slow_master_replica._message_req_service,
slow_master_replica._message_req_service.process_message_rep)
primary_node.sendToNodes(MessageRep(
**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {
f.INST_ID.nm: last_pp.instId,
f.VIEW_NO.nm: last_pp.viewNo,
f.PP_SEQ_NO.nm: last_pp.ppSeqNo
},
f.MSG.nm: last_pp
}),
names=[
slow_node.name,
])
def chk():
assert get_count(
slow_master_replica._message_req_service, slow_master_replica.
_message_req_service.process_message_rep) > count_pr_req
assert slow_master_replica._ordering_service.spylog.getLast(
"_validate").result[0] == -1
looper.run(eventually(chk, retryWait=1))
def send_bad(self, msg, frm):
if msg.msg_type == PREPREPARE:
resp = self.replicas[msg.params['instId']].getPrePrepare(
msg.params['viewNo'], msg.params['ppSeqNo'])
resp = updateNamedTuple(resp, digest='11908ffq')
self.sendToNodes(MessageRep(**{
f.MSG_TYPE.nm: msg.msg_type,
f.PARAMS.nm: msg.params,
f.MSG.nm: resp
}), names=[frm, ])
else:
return orig_method(msg, frm)
def test_process_message_rep_commit(message_req_service: MessageReqService, external_bus, data, commit):
key = (commit.viewNo, commit.ppSeqNo)
message_req_service.handlers[COMMIT].requested_messages[key] = None
message_rep = MessageRep(**{
f.MSG_TYPE.nm: COMMIT,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]},
f.MSG.nm: dict(commit.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(Commit, network_handler)
message_req_service.process_message_rep(message_rep, frm)
network_handler.assert_called_once_with(commit, frm)
def test_process_message_rep_new_view_from_primary(message_req_service: MessageReqService, external_bus, data,
new_view_message: NewView):
primary = data.primary_name
inst_id = data.inst_id
message_req_service.handlers[NEW_VIEW].requested_messages[data.view_no] = None
message_rep_from_primary = MessageRep(**{
f.MSG_TYPE.nm: NEW_VIEW,
f.PARAMS.nm: {f.INST_ID.nm: inst_id,
f.VIEW_NO.nm: data.view_no},
f.MSG.nm: dict(new_view_message.items())
})
network_handler = Mock()
external_bus.subscribe(NewView, network_handler)
message_req_service.process_message_rep(message_rep_from_primary, primary)
network_handler.assert_called_once_with(new_view_message, primary)
def test_process_message_rep_preprepare(message_req_service: MessageReqService, external_bus, data, pp):
key = (pp.viewNo, pp.ppSeqNo)
message_req_service.handlers[PREPREPARE].requested_messages[key] = None
message_rep = MessageRep(**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]},
f.MSG.nm: dict(pp.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(PrePrepare, network_handler)
message_req_service.process_message_rep(message_rep, frm)
network_handler.assert_called_once_with(pp, frm)
def test_process_message_req_preprepare(message_req_service: MessageReqService, external_bus, data, pp):
key = (pp.viewNo, pp.ppSeqNo)
message_req = MessageReq(**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]},
})
frm = "frm"
message_req_service.process_message_req(message_req, frm)
assert len(external_bus.sent_messages) == 1
assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type,
message_req.params,
data.sent_preprepares[key]),
[frm])
def process_message_req(self, msg: MessageReq, frm):
# Assumes a shared memory architecture. In case of multiprocessing,
# RPC architecture, use deques to communicate the message and node will
# maintain a unique internal message id to correlate responses.
msg_type = msg.msg_type
resp = self.req_handlers[msg_type](msg)
if resp is False:
return
self.sendToNodes(MessageRep(**{
f.MSG_TYPE.nm: msg_type,
f.PARAMS.nm: msg.params,
f.MSG.nm: resp
}), names=[frm, ])
def test_process_message_req_commit(message_req_service: MessageReqService, external_bus, data, commit):
key = (commit.viewNo, commit.ppSeqNo)
message_req = MessageReq(**{
f.MSG_TYPE.nm: COMMIT,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]},
})
frm = "frm"
message_req_service.process_message_req(message_req, frm)
assert len(external_bus.sent_messages) == 1
assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type,
message_req.params,
data.commits[key].msg),
[frm])
def process_message_req(self, msg: MessageReq, frm):
# Assumes a shared memory architecture. In case of multiprocessing,
# RPC architecture, use deques to communicate the message and node will
# maintain a unique internal message id to correlate responses.
msg_type = msg.msg_type
handler = self.handlers[msg_type]
resp = handler.serve(msg)
if not resp:
return
with self.metrics.measure_time(MetricsName.SEND_MESSAGE_REP_TIME):
self.sendToNodes(MessageRep(**{
f.MSG_TYPE.nm: msg_type,
f.PARAMS.nm: msg.params,
f.MSG.nm: resp
}), names=[frm, ])
def test_process_message_req_new_view(message_req_service: MessageReqService,
external_bus, data: ConsensusSharedData,
new_view_message: NewView):
frm = "frm"
data.primary_name = data.name
data.new_view_votes.add_new_view(new_view_message, data.primary_name)
message_req = MessageReq(**{
f.MSG_TYPE.nm: NEW_VIEW,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: data.view_no},
})
external_bus.process_incoming(message_req, frm)
assert len(external_bus.sent_messages) == 1
assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type,
message_req.params,
new_view_message._asdict()),
[frm])
def test_process_message_req_view_change(message_req_service: MessageReqService,
external_bus, data: ConsensusSharedData,
view_change_message: ViewChange):
frm = "frm"
digest = view_change_digest(view_change_message)
message_req = MessageReq(**{
f.MSG_TYPE.nm: VIEW_CHANGE,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.DIGEST.nm: digest,
f.NAME.nm: frm},
})
data.view_change_votes.add_view_change(view_change_message, frm)
external_bus.process_incoming(message_req, frm)
assert len(external_bus.sent_messages) == 1
assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type,
message_req.params,
view_change_message._asdict()),
[frm])
def test_process_message_rep_view_change_from_one(message_req_service: MessageReqService, external_bus, data,
view_change_message: ViewChange):
frm = "frm"
inst_id = data.inst_id
digest = view_change_digest(view_change_message)
key = (frm, digest)
message_req_service.handlers[VIEW_CHANGE].requested_messages[key] = None
message_rep_from_primary = MessageRep(**{
f.MSG_TYPE.nm: VIEW_CHANGE,
f.PARAMS.nm: {f.INST_ID.nm: inst_id,
f.DIGEST.nm: digest,
f.NAME.nm: frm},
f.MSG.nm: dict(view_change_message.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(ViewChange, network_handler)
network_handler.assert_not_called()
message_req_service.process_message_rep(message_rep_from_primary, frm)
network_handler.assert_called_once_with(view_change_message, frm)
def test_process_message_rep_with_incorrect_type(
message_req_service: MessageReqService, external_bus, data, pp):
key = (pp.viewNo, pp.ppSeqNo)
message_rep = MessageRep(
**{
f.MSG_TYPE.nm: LEDGER_STATUS,
f.PARAMS.nm: {
f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]
},
f.MSG.nm: dict(pp.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(PrePrepare, network_handler)
external_bus.subscribe(Prepare, network_handler)
external_bus.subscribe(Commit, network_handler)
message_req_service.process_message_rep(message_rep, frm)
network_handler.assert_not_called()
def test_process_message_rep_invalid_preprepare(
message_req_service: MessageReqService, external_bus, data, pp):
key = (pp.viewNo, pp.ppSeqNo)
msg_type = PREPREPARE
message_req_service.handlers[PREPREPARE].requested_messages[key] = None
message_rep = MessageRep(
**{
f.MSG_TYPE.nm: msg_type,
f.PARAMS.nm: {
f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]
},
f.MSG.nm: dict(pp.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(PrePrepare, network_handler)
message_req_service.handlers[
msg_type].extract_message = lambda msg, frm: raise_ex()
message_req_service.process_message_rep(message_rep, frm)
network_handler.assert_not_called()