def test_handle_delayed_preprepares(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ Make a node send PREPREPARE again after the slow node has ordered """ slow_node, other_nodes, primary_node, other_non_primary_nodes = \ split_nodes(txnPoolNodeSet) # This node will send PRE-PREPARE again orig_method = primary_node.handlers[PREPREPARE].serve last_pp = None def patched_method(self, msg): nonlocal last_pp last_pp = orig_method(msg) return last_pp primary_node.handlers[PREPREPARE].serve = types.MethodType(patched_method, primary_node.handlers[ PREPREPARE]) # Delay PRE-PREPAREs by large amount simulating loss slow_node.nodeIbStasher.delay(ppDelay(300, 0)) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=10, num_batches=5) waitNodeDataEquality(looper, slow_node, *other_nodes) slow_master_replica = slow_node.master_replica count_pr_req = get_count(slow_master_replica, slow_master_replica.process_requested_pre_prepare) count_pr_tpc = get_count(slow_master_replica, slow_master_replica.processThreePhaseMsg) primary_node.sendToNodes(MessageRep(**{ f.MSG_TYPE.nm: PREPREPARE, f.PARAMS.nm: { f.INST_ID.nm: last_pp.instId, f.VIEW_NO.nm: last_pp.viewNo, f.PP_SEQ_NO.nm: last_pp.ppSeqNo }, f.MSG.nm: last_pp }), names=[slow_node.name, ]) def chk(): # `process_requested_pre_prepare` is called but # `processThreePhaseMsg` is not called assert get_count( slow_master_replica, slow_master_replica.process_requested_pre_prepare) > count_pr_req assert get_count( slow_master_replica, slow_master_replica.processThreePhaseMsg) == count_pr_tpc looper.run(eventually(chk, retryWait=1))
def test_serialization_of_submessages_to_dict(): message = LedgerStatus(1, 10, None, None, "AwgQhPR9cgRubttBGjRruCRMLhZFBffbejbPipj7WBBm", CURRENT_PROTOCOL_VERSION) message_rep = MessageRep( **{ f.MSG_TYPE.nm: "LEDGER_STATUS", f.PARAMS.nm: { "ledger_id": 1, f.PROTOCOL_VERSION.nm: CURRENT_PROTOCOL_VERSION }, f.MSG.nm: message, }) serialized_message = ZStack.serializeMsg(message).decode() serialized_message_reply = ZStack.serializeMsg(message_rep).decode() # check that submessage (LedgerStatus) is serialized to the same dict as # it were a common message assert serialized_message in serialized_message_reply # check that de-serialized into the same message deserialized_message = LedgerStatus( **ZStack.deserializeMsg(serialized_message)) deserialized_submessage = LedgerStatus( **ZStack.deserializeMsg(serialized_message_reply)[f.MSG.nm]) assert message == deserialized_message assert message_rep.msg == deserialized_submessage assert message == deserialized_submessage
def process_message_req(self, msg: MessageReq, frm): # Assumes a shared memory architecture. In case of multiprocessing, # RPC architecture, use deques to communicate the message and node will # maintain a unique internal message id to correlate responses. msg_type = msg.msg_type if msg_type not in self.handlers.keys(): self.discard(msg, "Unknown message type {}".format(msg_type), self._logger.warning) return handler = self.handlers[msg_type] try: resp = handler.process_message_req(msg) except IncorrectMessageForHandlingException as e: self.discard(e.msg, e.reason, e.log_method) return if not resp: return with self.metrics.measure_time(MetricsName.SEND_MESSAGE_REP_TIME): self._network.send(MessageRep( **{ f.MSG_TYPE.nm: msg_type, f.PARAMS.nm: msg.params, f.MSG.nm: resp }), dst=[ frm.split(":")[0], ])
def test_handle_delayed_preprepares(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown, monkeypatch): """ Make a node send PREPREPARE again after the slow node has ordered """ slow_node, other_nodes, primary_node, other_non_primary_nodes = \ split_nodes(txnPoolNodeSet) # This node will send PRE-PREPARE again orig_method = primary_node.master_replica._message_req_service.handlers[ PREPREPARE].process_message_req handler = primary_node.master_replica._message_req_service.handlers[ PREPREPARE] last_pp = None def patched_method(self, msg): nonlocal last_pp last_pp = orig_method(msg) return last_pp handler.process_message_req = types.MethodType(patched_method, handler) # Delay PRE-PREPAREs by large amount simulating loss slow_node.nodeIbStasher.delay(ppDelay(300, 0)) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=10, num_batches=5) waitNodeDataEquality(looper, slow_node, *other_nodes) slow_master_replica = slow_node.master_replica count_pr_req = get_count( slow_master_replica._message_req_service, slow_master_replica._message_req_service.process_message_rep) primary_node.sendToNodes(MessageRep( **{ f.MSG_TYPE.nm: PREPREPARE, f.PARAMS.nm: { f.INST_ID.nm: last_pp.instId, f.VIEW_NO.nm: last_pp.viewNo, f.PP_SEQ_NO.nm: last_pp.ppSeqNo }, f.MSG.nm: last_pp }), names=[ slow_node.name, ]) def chk(): assert get_count( slow_master_replica._message_req_service, slow_master_replica. _message_req_service.process_message_rep) > count_pr_req assert slow_master_replica._ordering_service.spylog.getLast( "_validate").result[0] == -1 looper.run(eventually(chk, retryWait=1))
def send_bad(self, msg, frm): if msg.msg_type == PREPREPARE: resp = self.replicas[msg.params['instId']].getPrePrepare( msg.params['viewNo'], msg.params['ppSeqNo']) resp = updateNamedTuple(resp, digest='11908ffq') self.sendToNodes(MessageRep(**{ f.MSG_TYPE.nm: msg.msg_type, f.PARAMS.nm: msg.params, f.MSG.nm: resp }), names=[frm, ]) else: return orig_method(msg, frm)
def test_process_message_rep_commit(message_req_service: MessageReqService, external_bus, data, commit): key = (commit.viewNo, commit.ppSeqNo) message_req_service.handlers[COMMIT].requested_messages[key] = None message_rep = MessageRep(**{ f.MSG_TYPE.nm: COMMIT, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1]}, f.MSG.nm: dict(commit.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(Commit, network_handler) message_req_service.process_message_rep(message_rep, frm) network_handler.assert_called_once_with(commit, frm)
def test_process_message_rep_new_view_from_primary(message_req_service: MessageReqService, external_bus, data, new_view_message: NewView): primary = data.primary_name inst_id = data.inst_id message_req_service.handlers[NEW_VIEW].requested_messages[data.view_no] = None message_rep_from_primary = MessageRep(**{ f.MSG_TYPE.nm: NEW_VIEW, f.PARAMS.nm: {f.INST_ID.nm: inst_id, f.VIEW_NO.nm: data.view_no}, f.MSG.nm: dict(new_view_message.items()) }) network_handler = Mock() external_bus.subscribe(NewView, network_handler) message_req_service.process_message_rep(message_rep_from_primary, primary) network_handler.assert_called_once_with(new_view_message, primary)
def test_process_message_rep_preprepare(message_req_service: MessageReqService, external_bus, data, pp): key = (pp.viewNo, pp.ppSeqNo) message_req_service.handlers[PREPREPARE].requested_messages[key] = None message_rep = MessageRep(**{ f.MSG_TYPE.nm: PREPREPARE, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1]}, f.MSG.nm: dict(pp.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(PrePrepare, network_handler) message_req_service.process_message_rep(message_rep, frm) network_handler.assert_called_once_with(pp, frm)
def test_process_message_req_preprepare(message_req_service: MessageReqService, external_bus, data, pp): key = (pp.viewNo, pp.ppSeqNo) message_req = MessageReq(**{ f.MSG_TYPE.nm: PREPREPARE, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1]}, }) frm = "frm" message_req_service.process_message_req(message_req, frm) assert len(external_bus.sent_messages) == 1 assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type, message_req.params, data.sent_preprepares[key]), [frm])
def process_message_req(self, msg: MessageReq, frm): # Assumes a shared memory architecture. In case of multiprocessing, # RPC architecture, use deques to communicate the message and node will # maintain a unique internal message id to correlate responses. msg_type = msg.msg_type resp = self.req_handlers[msg_type](msg) if resp is False: return self.sendToNodes(MessageRep(**{ f.MSG_TYPE.nm: msg_type, f.PARAMS.nm: msg.params, f.MSG.nm: resp }), names=[frm, ])
def test_process_message_req_commit(message_req_service: MessageReqService, external_bus, data, commit): key = (commit.viewNo, commit.ppSeqNo) message_req = MessageReq(**{ f.MSG_TYPE.nm: COMMIT, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1]}, }) frm = "frm" message_req_service.process_message_req(message_req, frm) assert len(external_bus.sent_messages) == 1 assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type, message_req.params, data.commits[key].msg), [frm])
def process_message_req(self, msg: MessageReq, frm): # Assumes a shared memory architecture. In case of multiprocessing, # RPC architecture, use deques to communicate the message and node will # maintain a unique internal message id to correlate responses. msg_type = msg.msg_type handler = self.handlers[msg_type] resp = handler.serve(msg) if not resp: return with self.metrics.measure_time(MetricsName.SEND_MESSAGE_REP_TIME): self.sendToNodes(MessageRep(**{ f.MSG_TYPE.nm: msg_type, f.PARAMS.nm: msg.params, f.MSG.nm: resp }), names=[frm, ])
def test_process_message_req_new_view(message_req_service: MessageReqService, external_bus, data: ConsensusSharedData, new_view_message: NewView): frm = "frm" data.primary_name = data.name data.new_view_votes.add_new_view(new_view_message, data.primary_name) message_req = MessageReq(**{ f.MSG_TYPE.nm: NEW_VIEW, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: data.view_no}, }) external_bus.process_incoming(message_req, frm) assert len(external_bus.sent_messages) == 1 assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type, message_req.params, new_view_message._asdict()), [frm])
def test_process_message_req_view_change(message_req_service: MessageReqService, external_bus, data: ConsensusSharedData, view_change_message: ViewChange): frm = "frm" digest = view_change_digest(view_change_message) message_req = MessageReq(**{ f.MSG_TYPE.nm: VIEW_CHANGE, f.PARAMS.nm: {f.INST_ID.nm: data.inst_id, f.DIGEST.nm: digest, f.NAME.nm: frm}, }) data.view_change_votes.add_view_change(view_change_message, frm) external_bus.process_incoming(message_req, frm) assert len(external_bus.sent_messages) == 1 assert external_bus.sent_messages[0] == (MessageRep(message_req.msg_type, message_req.params, view_change_message._asdict()), [frm])
def test_process_message_rep_view_change_from_one(message_req_service: MessageReqService, external_bus, data, view_change_message: ViewChange): frm = "frm" inst_id = data.inst_id digest = view_change_digest(view_change_message) key = (frm, digest) message_req_service.handlers[VIEW_CHANGE].requested_messages[key] = None message_rep_from_primary = MessageRep(**{ f.MSG_TYPE.nm: VIEW_CHANGE, f.PARAMS.nm: {f.INST_ID.nm: inst_id, f.DIGEST.nm: digest, f.NAME.nm: frm}, f.MSG.nm: dict(view_change_message.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(ViewChange, network_handler) network_handler.assert_not_called() message_req_service.process_message_rep(message_rep_from_primary, frm) network_handler.assert_called_once_with(view_change_message, frm)
def test_process_message_rep_with_incorrect_type( message_req_service: MessageReqService, external_bus, data, pp): key = (pp.viewNo, pp.ppSeqNo) message_rep = MessageRep( **{ f.MSG_TYPE.nm: LEDGER_STATUS, f.PARAMS.nm: { f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1] }, f.MSG.nm: dict(pp.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(PrePrepare, network_handler) external_bus.subscribe(Prepare, network_handler) external_bus.subscribe(Commit, network_handler) message_req_service.process_message_rep(message_rep, frm) network_handler.assert_not_called()
def test_process_message_rep_invalid_preprepare( message_req_service: MessageReqService, external_bus, data, pp): key = (pp.viewNo, pp.ppSeqNo) msg_type = PREPREPARE message_req_service.handlers[PREPREPARE].requested_messages[key] = None message_rep = MessageRep( **{ f.MSG_TYPE.nm: msg_type, f.PARAMS.nm: { f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1] }, f.MSG.nm: dict(pp.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(PrePrepare, network_handler) message_req_service.handlers[ msg_type].extract_message = lambda msg, frm: raise_ex() message_req_service.process_message_rep(message_rep, frm) network_handler.assert_not_called()