def test_process_ordered(checkpoint_service, ordered, pre_prepare, tconf): with pytest.raises( LogicError, match="CheckpointService | Can't process Ordered msg because " "ppSeqNo {} not in preprepared".format(ordered.ppSeqNo)): checkpoint_service.process_ordered(ordered) checkpoint_service._data.preprepared.append( preprepare_to_batch_id(pre_prepare)) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, tconf.CHK_FREQ, pre_prepare) pre_prepare.ppSeqNo = tconf.CHK_FREQ ordered.ppSeqNo = pre_prepare.ppSeqNo checkpoint_service._data.preprepared.append( preprepare_to_batch_id(pre_prepare)) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, tconf.CHK_FREQ, pre_prepare, check_shared_data=True) pre_prepare.ppSeqNo += 1 ordered.ppSeqNo = pre_prepare.ppSeqNo checkpoint_service._data.preprepared.append( preprepare_to_batch_id(pre_prepare)) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, tconf.CHK_FREQ * 2, pre_prepare)
def test_send_reply_on_old_view_pre_prepares_request( external_bus, orderer, initial_view_no, stored_old_view_pre_prepares, requested_old_view_pre_prepares): # Setup orderer._data.view_no = initial_view_no + 2 orderer._update_old_view_preprepares(stored_old_view_pre_prepares) # Receive OldViewPrePrepareRequest req batches = [ preprepare_to_batch_id(pp) for pp in requested_old_view_pre_prepares ] req = OldViewPrePrepareRequest(0, batches) frm = "node1" orderer._network.process_incoming(req, generateName(frm, orderer._data.inst_id)) # Check that OldViewPrePrepareReply is sent for all requested PrePrepares if not orderer.is_master: assert len(external_bus.sent_messages) == 0 return # equal to set's union operation expected_pps = [ i for i in stored_old_view_pre_prepares if i in requested_old_view_pre_prepares ] expected_pps = sorted(expected_pps, key=lambda pp: pp.ppSeqNo) check_reply_old_view_preprepares_sent(external_bus, frm, expected_pps)
def test_node_reg_in_ordered_from_audit_for_tree_txns(test_node): node_regs = {} replica = test_node.master_replica node_reg = ["Alpha", "Beta", "Gamma", "Delta", "Eta"] for i in range(3): pp = create_pre_prepare_no_bls(state_root=generate_state_root(), pp_seq_no=i) key = (pp.viewNo, pp.ppSeqNo) replica._ordering_service.prePrepares[key] = pp replica._consensus_data.preprepared.append(preprepare_to_batch_id(pp)) three_pc_batch = ThreePcBatch.from_pre_prepare( pre_prepare=pp, state_root=pp.stateRootHash, txn_root=pp.txnRootHash, valid_digests=pp.reqIdr) three_pc_batch.node_reg = node_reg + ["Node{}".format(i + 10)] three_pc_batch.primaries = ["Alpha", "Beta"] test_node.write_manager.audit_b_handler.post_batch_applied( three_pc_batch) node_regs[key] = three_pc_batch.node_reg for key in reversed(list(node_regs.keys())): pp = replica._ordering_service.get_preprepare(*key) assert replica._ordering_service._get_node_reg_for_ordered( pp) == node_regs[key]
def test_re_order_pre_prepares_no_pre_prepares(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): # 1. drop PrePrepars, Prepares and Commits on 4thNode # Order a couple of requests on Nodes 1-3 lagging_node = txnPoolNodeSet[-1] other_nodes = txnPoolNodeSet[:-1] with delay_rules_without_processing(lagging_node.nodeIbStasher, delay_3pc()): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3) assert all(n.master_last_ordered_3PC == (0, 3) for n in other_nodes) with delay_rules_without_processing(lagging_node.nodeIbStasher, msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])): # 2. simulate view change start so that # all PrePrepares/Prepares/Commits are cleared # and uncommitted txns are reverted for n in txnPoolNodeSet: n.replicas.send_to_internal_bus(ViewChangeStarted(view_no=1)) master_ordering_service = n.master_replica._ordering_service assert not master_ordering_service.prePrepares assert not master_ordering_service.prepares assert not master_ordering_service.commits ledger = n.db_manager.ledgers[DOMAIN_LEDGER_ID] state = n.db_manager.states[DOMAIN_LEDGER_ID] assert len(ledger.uncommittedTxns) == 0 assert ledger.uncommitted_root_hash == ledger.tree.root_hash assert state.committedHead == state.head # check that all nodes but the lagging one have old_view_pps stored for n in other_nodes: assert n.master_replica._ordering_service.old_view_preprepares assert not lagging_node.master_replica._ordering_service.old_view_preprepares # 3. Simulate View Change finish to re-order the same PrePrepare assert lagging_node.master_last_ordered_3PC == (0, 0) new_master = txnPoolNodeSet[1] batches = sorted([preprepare_to_batch_id(pp) for _, pp in new_master.master_replica._ordering_service.old_view_preprepares.items()]) new_view_msg = NewView(viewNo=0, viewChanges=[], checkpoint=None, batches=batches) new_view_chk_applied_msg = NewViewCheckpointsApplied(view_no=0, view_changes=[], checkpoint=None, batches=batches) for n in txnPoolNodeSet: n.master_replica._consensus_data.new_view = new_view_msg n.master_replica._consensus_data.prev_view_prepare_cert = batches[-1].pp_seq_no n.master_replica._ordering_service._bus.send(new_view_chk_applied_msg) # 4. Make sure that the nodes 1-3 (that already ordered the requests) sent Prepares and Commits so that # the request was eventually ordered on Node4 as well waitNodeDataEquality(looper, lagging_node, *other_nodes, customTimeout=60) assert lagging_node.master_last_ordered_3PC == (0, 4) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf, ordered, validators, is_master): checkpoint_stabilized_handler = Mock() checkpoint_service._bus.subscribe(CheckpointStabilized, checkpoint_stabilized_handler) quorum = checkpoint_service._data.quorums.checkpoint.value n = len(validators) assert quorum == n - getMaxFailures(n) - 1 senders = ["sender{}".format(i) for i in range(quorum + 1)] till_seq_no = tconf.CHK_FREQ checkpoint_service._received_checkpoints[cp_key(checkpoint.viewNo, 1)] = {"frm"} # For now, on checkpoint stabilization phase all checkpoints # with ppSeqNo less then stable_checkpoint will be removed checkpoint_service._received_checkpoints[cp_key( checkpoint.viewNo + 1, till_seq_no + 100)] = {"frm"} pre_prepare.ppSeqNo = till_seq_no pre_prepare.auditTxnRootHash = cp_digest(till_seq_no) ordered.ppSeqNo = pre_prepare.ppSeqNo ordered.auditTxnRootHash = pre_prepare.auditTxnRootHash checkpoint_service._data.preprepared.append( preprepare_to_batch_id(pre_prepare)) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, till_seq_no, pre_prepare, check_shared_data=True) for sender in senders[:quorum - 1]: checkpoint_service.process_checkpoint(checkpoint, sender) assert checkpoint_service._data.stable_checkpoint < till_seq_no # send the last checkpoint to stable it checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1]) assert checkpoint_service._data.stable_checkpoint == till_seq_no # check _remove_stashed_checkpoints() assert sum(1 for cp in checkpoint_service._received_checkpoints if cp.view_no == checkpoint.viewNo) == 0 assert sum(1 for cp in checkpoint_service._received_checkpoints if cp.view_no == checkpoint.viewNo + 1) > 0 # check watermarks assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd # check that a Cleanup msg has been sent checkpoint_stabilized_handler.assert_called_once_with( CheckpointStabilized(last_stable_3pc=(checkpoint.viewNo, checkpoint.seqNoEnd)))
def test_cleanup_after_checkpoint_stabilize(orderer): pre_prepares = [ create_pre_prepare_no_bls(generate_state_root(), view_no=0, pp_seq_no=1), create_pre_prepare_no_bls(generate_state_root(), view_no=1, pp_seq_no=2), create_pre_prepare_no_bls(generate_state_root(), view_no=1, pp_seq_no=3) ] dicts_to_cleaning = [ orderer.pre_prepare_tss, orderer.sent_preprepares, orderer.prePrepares, orderer.prepares, orderer.commits, orderer.batches, orderer.pre_prepares_stashed_for_incorrect_time ] lists_to_cleaning = [orderer._data.prepared, orderer._data.preprepared] for pp in pre_prepares: for dict_to_cleaning in dicts_to_cleaning: dict_to_cleaning[(pp.viewNo, pp.ppSeqNo)] = pp for list_to_cleaning in lists_to_cleaning: list_to_cleaning.append(preprepare_to_batch_id(pp)) orderer._bus.send(CheckpointStabilized(last_stable_3pc=(1, 2))) for pp in pre_prepares[:2]: for dict_to_cleaning in dicts_to_cleaning: assert (pp.viewNo, pp.ppSeqNo) not in dict_to_cleaning for list_to_cleaning in lists_to_cleaning: assert preprepare_to_batch_id(pp) not in list_to_cleaning for dict_to_cleaning in dicts_to_cleaning: assert (pre_prepares[2].viewNo, pre_prepares[2].ppSeqNo) in dict_to_cleaning for list_to_cleaning in lists_to_cleaning: assert preprepare_to_batch_id(pre_prepares[2]) in list_to_cleaning
def test_node_reg_in_ordered_from_audit(test_node): pre_prepare = create_pre_prepare_no_bls(state_root=generate_state_root(), pp_seq_no=1) replica = test_node.master_replica key = (pre_prepare.viewNo, pre_prepare.ppSeqNo) replica._ordering_service.prePrepares[key] = pre_prepare replica._consensus_data.preprepared.append( preprepare_to_batch_id(pre_prepare)) three_pc_batch = ThreePcBatch.from_pre_prepare( pre_prepare=pre_prepare, state_root=pre_prepare.stateRootHash, txn_root=pre_prepare.txnRootHash, valid_digests=pre_prepare.reqIdr) three_pc_batch.node_reg = ["Alpha", "Beta", "Gamma", "Delta", "Eta"] three_pc_batch.primaries = ["Alpha", "Beta"] test_node.write_manager.audit_b_handler.post_batch_applied(three_pc_batch) assert replica._ordering_service._get_node_reg_for_ordered( pre_prepare) == three_pc_batch.node_reg
def test_process_message_rep_already_ordered_preprepare( message_req_service: MessageReqService, external_bus, data, pp): key = (pp.viewNo, pp.ppSeqNo) data.preprepared.append(preprepare_to_batch_id(pp)) message_req_service.handlers[PREPREPARE].requested_messages[key] = None message_rep = MessageRep( **{ f.MSG_TYPE.nm: PREPREPARE, f.PARAMS.nm: { f.INST_ID.nm: data.inst_id, f.VIEW_NO.nm: key[0], f.PP_SEQ_NO.nm: key[1] }, f.MSG.nm: dict(pp.items()) }) frm = "frm" network_handler = Mock() external_bus.subscribe(PrePrepare, network_handler) message_req_service.process_message_rep(message_rep, frm) network_handler.assert_not_called()
def create_batches_from_preprepares(preprepares): return [preprepare_to_batch_id(pp) for pp in preprepares]
def test_process_preprepare_on_new_view_checkpoint_applied( internal_bus, external_bus, orderer, is_primary, all_ordered, initial_view_no, pre_prepares, stored_old_view_pre_prepares): # !!!SETUP!!! orderer._data.view_no = initial_view_no + 1 batches = create_batches_from_preprepares(pre_prepares) orderer._data.prev_view_prepare_cert = batches[-1].pp_seq_no new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200, batches=batches) # emulate that we received all PrePrepares before View Change orderer._update_old_view_preprepares(stored_old_view_pre_prepares) # emulate that we've already ordered the PrePrepares if all_ordered and stored_old_view_pre_prepares: orderer.last_ordered_3pc = (initial_view_no, stored_old_view_pre_prepares[-1].ppSeqNo) # !!!EXECUTE!!! # send NewViewCheckpointsApplied internal_bus.send( NewViewCheckpointsApplied(view_no=initial_view_no + 1, view_changes=new_view.viewChanges, checkpoint=new_view.checkpoint, batches=new_view.batches)) # !!!CHECK!!! if not orderer.is_master: # no re-ordering is expected on non-master assert orderer._data.preprepared == [] assert orderer._data.prepared == [] return # check that PPs were added stored_batch_ids = [ preprepare_to_batch_id(pp) for pp in stored_old_view_pre_prepares ] assert orderer._data.preprepared == [ BatchID(view_no=initial_view_no + 1, pp_view_no=initial_view_no, pp_seq_no=batch_id.pp_seq_no, pp_digest=batch_id.pp_digest) for batch_id in new_view.batches if batch_id in stored_batch_ids ] # check that sentPrePrepares is updated in case of Primary and prePrepares in case of non-primary updated_prepares_collection = orderer.prePrepares if not is_primary else orderer.sent_preprepares non_updated_prepares_collection = orderer.sent_preprepares if not is_primary else orderer.prePrepares for pp in stored_old_view_pre_prepares: new_pp = updateNamedTuple(pp, viewNo=initial_view_no + 1, originalViewNo=pp.viewNo) assert (initial_view_no + 1, new_pp.ppSeqNo) in updated_prepares_collection assert updated_prepares_collection[(initial_view_no + 1, new_pp.ppSeqNo)] == new_pp assert not non_updated_prepares_collection # check that Prepare is sent in case of non primary if not is_primary: check_prepares_sent(external_bus, stored_old_view_pre_prepares, initial_view_no + 1) else: # only MessageReqs are sent assert len(external_bus.sent_messages ) == len(pre_prepares) - len(stored_old_view_pre_prepares) # we don't have a quorum of Prepares yet assert orderer._data.prepared == [] # check that missing PrePrepares have been requested expected_requested_batches = [ batch_id for batch_id in new_view.batches if batch_id not in stored_batch_ids ] check_request_old_view_preprepares_sent(external_bus, expected_requested_batches)