def test_get_msgs_from_rxMsgs_queue(create_node_and_not_start, looper): node = create_node_and_not_start node.view_changer = ViewChanger(node) node.view_changer.view_no = 0 """pre_view_change stage""" node.view_changer.startViewChange(1) assert node.view_changer.view_no == 0 prepare = Prepare( 0, 0, 1, get_utc_epoch(), 'f99937241d4c891c08e92a3cc25966607315ca66b51827b170d492962d58a9be', 'CZecK1m7VYjSNCC7pGHj938DSW2tfbqoJp1bMJEtFqvG', '7WrAMboPTcMaQCU1raoj28vnhu2bPMMd2Lr9tEcsXeCJ') inst_change = InstanceChange(1, 25) m = node.nodeInBox.popleft() assert isinstance(m[0], ViewChangeStartMessage) node.nodestack.addRemote('someNode', genHa(), b'1DYuELN<SHbv1?NJ=][4De%^Hge887B0I!s<YGdD', 'pubkey') node.nodestack.rxMsgs.append((json.dumps(prepare._asdict()), 'pubkey')) node.nodestack.rxMsgs.append((json.dumps(inst_change._asdict()), 'pubkey')) node.msgHasAcceptableViewNo = lambda *args, **kwargs: True """While processing ViewChangeStartMessage from nodeInBox queue, should be: - move msgs from rxMsgs queue to nodeInBox queue - process all 3PC msgs (for Prepare msg it should be moved to inBox queue of master_replica) - add ViewChangeContinue msg into master_replica's inBox queue - all not 3PC msgs will be stashed in strategy queue""" looper.run(node.process_one_node_message(m)) m = node.master_replica.inBox.popleft() assert isinstance(m[0], Prepare) m = node.master_replica.inBox.popleft() assert isinstance(m, ViewChangeContinueMessage) m = node.view_changer.pre_vc_strategy.stashedNodeInBox.popleft() assert isinstance(m[0], InstanceChange)
def test_get_msgs_from_rxMsgs_queue(create_node_and_not_start, looper): node = create_node_and_not_start node.view_changer = create_view_changer(node) node.view_changer.pre_vc_strategy = VCStartMsgStrategy(view_changer, node) node.view_changer.view_no = 0 """pre_view_change stage""" node.view_changer.start_view_change(1) assert node.view_changer.view_no == 0 prepare = Prepare( 0, 0, 1, get_utc_epoch(), 'f99937241d4c891c08e92a3cc25966607315ca66b51827b170d492962d58a9be', 'CZecK1m7VYjSNCC7pGHj938DSW2tfbqoJp1bMJEtFqvG', '7WrAMboPTcMaQCU1raoj28vnhu2bPMMd2Lr9tEcsXeCJ') inst_change = InstanceChange(1, 25) m = node.nodeInBox.popleft() assert isinstance(m[0], ViewChangeStartMessage) node.nodestack.addRemote('someNode', genHa(), b'1DYuELN<SHbv1?NJ=][4De%^Hge887B0I!s<YGdD', 'pubkey') node.nodestack.rxMsgs.append((json.dumps(prepare._asdict()), 'pubkey')) node.nodestack.rxMsgs.append((json.dumps(inst_change._asdict()), 'pubkey')) node.msgHasAcceptableViewNo = lambda *args, **kwargs: True """While processing ViewChangeStartMessage from nodeInBox queue, should be: - move msgs from rxMsgs queue to nodeInBox queue - process all 3PC msgs (for Prepare msg it should be moved to inBox queue of master_replica) - add ViewChangeContinue msg into master_replica's inBox queue - all not 3PC msgs will be stashed in strategy queue""" looper.run(node.process_one_node_message(m)) m = node.master_replica.inBox.popleft() assert isinstance(m[0], Prepare) m = node.master_replica.inBox.popleft() assert isinstance(m, ViewChangeContinueMessage) m = node.view_changer.pre_vc_strategy.stashedNodeInBox.popleft() assert isinstance(m[0], InstanceChange)
def createInstanceChangeMessage(): # Creating a message this way to exclude # client-side validation of viewNo goodViewNo = 1 badViewNo = "BAD" icMsg = InstanceChange(viewNo=goodViewNo, reason=0) icMsg._fields["viewNo"] = badViewNo return icMsg
def test_the_same_order_as_in_NodeInBox_after_vc_continued(pre_vc_strategy): replica = pre_vc_strategy.replica pre_vc_strategy.view_changer.view_no = 1 m1 = (InstanceChange(3, 25), "Beta") m2 = (InstanceChange(3, 25), "Gamma") pre_vc_strategy.stashedNodeInBox.append(m1) pre_vc_strategy.stashedNodeInBox.append(m2) assert len(pre_vc_strategy.node.nodeInBox) == 0 pre_vc_strategy.on_view_change_continued(replica, ViewChangeContinueMessage(2)) assert len(pre_vc_strategy.node.nodeInBox) > 0 assert pre_vc_strategy.node.nodeInBox.popleft() == m1 assert pre_vc_strategy.node.nodeInBox.popleft() == m2
def test_stash_not_3PC_msgs(pre_vc_strategy, looper): node = pre_vc_strategy.node pre_vc_strategy.view_changer.view_no = 1 m1 = (InstanceChange(3, 25), "Beta") m2 = (InstanceChange(3, 25), "Gamma") assert len(pre_vc_strategy.stashedNodeInBox) == 0 node.nodeInBox.append(m1) node.nodeInBox.append(m2) looper.run(pre_vc_strategy.on_view_change_started(pre_vc_strategy.node, ViewChangeStartMessage(2), "some_node")) assert len(pre_vc_strategy.stashedNodeInBox) > 0 """In the same order as from nodeInBox queue""" assert pre_vc_strategy.stashedNodeInBox.popleft() == m1 assert pre_vc_strategy.stashedNodeInBox.popleft() == m2
def _propose_view_change(self, suspision_code): proposed_view_no = self._data.view_no + 1 self._logger.info( "{} proposing a view change to {} with code {}".format( self, proposed_view_no, suspision_code)) msg = InstanceChange(proposed_view_no, suspision_code) self._network.send(msg)
def _send_instance_change(self, view_no: int, suspicion: Suspicion): logger.info("{}{} sending an instance change with view_no {} since {}". format(VIEW_CHANGE_PREFIX, self, view_no, suspicion.reason)) msg = InstanceChange(view_no, suspicion.code) self._network.send(msg) # record instance change vote for self and try to change the view if quorum is reached self._on_verified_instance_change_msg(msg, self.name)
def test_instance_change_from_known(view_change_trigger_service): current_view = view_change_trigger_service._data.view_no proposed_view = current_view + 1 ic_msg = InstanceChange(viewNo=proposed_view, reason=26) frm = list(view_change_trigger_service._data.validators)[0] view_change_trigger_service.process_instance_change(ic_msg, frm=frm) assert view_change_trigger_service._instance_changes.has_inst_chng_from( proposed_view, frm)
def test_instance_change_from_unknown(view_change_trigger_service): current_view = view_change_trigger_service._data.view_no proposed_view = current_view + 1 ic_msg = InstanceChange(viewNo=proposed_view, reason=26) frm = 'SomeUnknownNode' view_change_trigger_service.process_instance_change(ic_msg, frm=frm) assert not view_change_trigger_service._instance_changes.has_inst_chng_from( proposed_view, frm)
def test_instance_changes_has_quorum_when_enough_distinct_votes_are_added(instance_changes): quorum = 2 view_no = 1 assert not instance_changes.has_quorum(view_no, quorum) for i in range(quorum): instance_changes.add_vote(InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code), "Node{}".format(i)) assert instance_changes.has_quorum(view_no, quorum)
def _propose_view_change(self, suspision_code): proposed_view_no = self._data.view_no # TODO: For some reason not incrementing view_no in most cases leads to lots of failing/flaky tests # if suspicion == Suspicions.INSTANCE_CHANGE_TIMEOUT or not self.view_change_in_progress: if suspision_code != Suspicions.STATE_SIGS_ARE_NOT_UPDATED or not self._data.waiting_for_new_view: proposed_view_no += 1 self._logger.info( "{} proposing a view change to {} with code {}".format( self, proposed_view_no, suspision_code)) msg = InstanceChange(proposed_view_no, suspision_code) self._network.send(msg)
def test_add_first_vote(instance_changes): frm = "Node1" view_no = 1 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) assert not instance_changes instance_changes.add_vote(msg, frm) assert instance_changes[view_no].msg == msg assert instance_changes[view_no].voters[frm] assert instance_changes.has_view(view_no) assert instance_changes.has_inst_chng_from(view_no, frm)
def test_remove_view(instance_change_provider): frm = "Node1" view_no = 2 instance_change_provider.add_vote( InstanceChange(view_no - 1, Suspicions.PRIMARY_DEGRADED.code), frm) instance_change_provider.add_vote( InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code), frm) assert instance_change_provider.has_view(view_no - 1) assert instance_change_provider.has_view(view_no) assert instance_change_provider.has_inst_chng_from(view_no - 1, frm) assert instance_change_provider.has_inst_chng_from(view_no, frm) assert _is_view_in_db(view_no, instance_change_provider) instance_change_provider.remove_view(view_no) assert not instance_change_provider.has_view(view_no - 1) assert not instance_change_provider.has_view(view_no) assert not instance_change_provider.has_inst_chng_from(view_no - 1, frm) assert not instance_change_provider.has_inst_chng_from(view_no, frm) assert not _is_view_in_db(view_no, instance_change_provider)
def test_add_first_vote(instance_change_provider): frm = "Node1" view_no = 1 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) assert not instance_change_provider.has_view(view_no) assert not instance_change_provider.has_inst_chng_from(view_no, frm) instance_change_provider.add_vote(msg, frm) assert instance_change_provider.has_view(view_no) assert instance_change_provider.has_inst_chng_from(view_no, frm) assert _is_view_in_db(view_no, instance_change_provider)
def test_add_vc_start_msg_during_start_view_change(txnPoolNodeSet, looper): delayed_node = txnPoolNodeSet[-1] current_view_no = delayed_node.viewNo proposed_view_no = current_view_no + 1 delayed_node.nodeIbStasher.delay(vcd_delay(1000)) delayed_node.nodeIbStasher.delay(icDelay(1000)) assert delayed_node.view_changer ensure_view_change(looper, txnPoolNodeSet[:-1]) """ If view number was incremented, that means that instanceChange quorum was archived """ assert txnPoolNodeSet[0].viewNo == proposed_view_no looper.removeProdable(delayed_node) delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Alpha')) delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Beta')) delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Gamma')) delayed_node.processNodeInBox = functools.partial(Node.processNodeInBox, delayed_node) looper.run(delayed_node.processNodeInBox()) looper.run(delayed_node.serviceViewChanger(None)) assert len(delayed_node.nodeInBox) == 1 m = delayed_node.nodeInBox.popleft() assert isinstance(m[0], ViewChangeStartMessage)
def test_equal_votes_dont_accumulate_when_added(instance_changes, tconf): frm = "Node1" view_no = 1 time_provider = MockTimestamp(0) second_vote_time = 1 instance_changes = InstanceChanges(tconf, time_provider) msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) instance_changes.add_vote(msg, frm) time_provider.value = second_vote_time instance_changes.add_vote(msg, frm) assert instance_changes[view_no].voters[frm] == second_vote_time assert len(instance_changes[view_no].voters) == 1 assert len(instance_changes) == 1
def test_old_ic_discard(instance_change_provider, tconf, time_provider): frm = "Node1" view_no = 1 quorum = 2 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) time_provider.value = 0 instance_change_provider.add_vote(msg, frm) time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1 assert not instance_change_provider.has_view(view_no) instance_change_provider.add_vote(msg, frm) time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1 assert not instance_change_provider.has_inst_chng_from(view_no, frm) instance_change_provider.add_vote(msg, frm) time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1 assert not instance_change_provider.has_quorum(view_no, quorum)
def test_equal_votes_dont_accumulate_when_added(instance_change_provider, time_provider): frm = "Node1" view_no = 1 quorum = 2 time_provider.value = 0 second_vote_time = 1 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) instance_change_provider.add_vote(msg, frm) time_provider.value = second_vote_time instance_change_provider.add_vote(msg, frm) assert instance_change_provider.has_view(view_no) assert instance_change_provider.has_inst_chng_from(view_no, frm) assert not instance_change_provider.has_quorum(view_no, quorum) instance_changes_db = instance_change_provider._node_status_db.get( instance_change_provider.generate_db_key(view_no)) assert len(node_status_db_serializer.deserialize(instance_changes_db)) == 1
def test_too_old_messages_dont_count_towards_quorum(instance_changes, tconf): frm1 = "Node1" frm2 = "Node2" view_no = 1 quorum = 2 time_provider = MockTimestamp(0) instance_changes = InstanceChanges(tconf, time_provider) msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) instance_changes.add_vote(msg, frm1) time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2) instance_changes.add_vote(msg, frm2) time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2) + 1 assert not instance_changes.has_quorum(view_no, quorum) assert instance_changes.has_view(view_no) assert instance_changes[view_no].msg == msg assert not instance_changes.has_inst_chng_from(view_no, frm1) assert instance_changes.has_inst_chng_from(view_no, frm2)
def test_too_old_messages_dont_count_towards_quorum(instance_change_provider, time_provider, tconf): frm1 = "Node1" frm2 = "Node2" view_no = 1 quorum = 2 time_provider.value = 0 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) instance_change_provider.add_vote(msg, frm1) time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL / 2) instance_change_provider.add_vote(msg, frm2) assert instance_change_provider.has_quorum(view_no, quorum) time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL / 2) + 1 assert not instance_change_provider.has_quorum(view_no, quorum) assert not instance_change_provider.has_inst_chng_from(view_no, frm1) assert instance_change_provider.has_inst_chng_from(view_no, frm2) instance_changes_db = instance_change_provider._node_status_db.get( instance_change_provider.generate_db_key(view_no)) assert len(node_status_db_serializer.deserialize(instance_changes_db)) == 1
def test_update_instance_changes_in_db(instance_change_provider, tconf, node_status_db, time_provider): frm = "Node1" view_no = 1 msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code) assert not instance_change_provider.has_view(view_no) assert not instance_change_provider.has_inst_chng_from(view_no, frm) assert not _is_view_in_db(view_no, instance_change_provider) instance_change_provider.add_vote(msg, frm) assert instance_change_provider.has_view(view_no) assert instance_change_provider.has_inst_chng_from(view_no, frm) assert _is_view_in_db(view_no, instance_change_provider) instance_change_provider._node_status_db.close() assert instance_change_provider._node_status_db.closed instance_change_provider._node_status_db.open() new_instance_change_provider = InstanceChangeProvider( tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL, node_status_db, time_provider) assert new_instance_change_provider.has_view(view_no) assert new_instance_change_provider.has_inst_chng_from(view_no, frm)
def _create_instance_change_msg(self, view_no, suspicion_code): return InstanceChange(view_no, suspicion_code)