def test_get_msgs_from_rxMsgs_queue(create_node_and_not_start, looper):
node = create_node_and_not_start
node.view_changer = ViewChanger(node)
node.view_changer.view_no = 0
"""pre_view_change stage"""
node.view_changer.startViewChange(1)
assert node.view_changer.view_no == 0
prepare = Prepare(
0,
0,
1,
get_utc_epoch(),
'f99937241d4c891c08e92a3cc25966607315ca66b51827b170d492962d58a9be',
'CZecK1m7VYjSNCC7pGHj938DSW2tfbqoJp1bMJEtFqvG',
'7WrAMboPTcMaQCU1raoj28vnhu2bPMMd2Lr9tEcsXeCJ')
inst_change = InstanceChange(1, 25)
m = node.nodeInBox.popleft()
assert isinstance(m[0], ViewChangeStartMessage)
node.nodestack.addRemote('someNode', genHa(), b'1DYuELN<SHbv1?NJ=][4De%^Hge887B0I!s<YGdD', 'pubkey')
node.nodestack.rxMsgs.append((json.dumps(prepare._asdict()), 'pubkey'))
node.nodestack.rxMsgs.append((json.dumps(inst_change._asdict()), 'pubkey'))
node.msgHasAcceptableViewNo = lambda *args, **kwargs: True
"""While processing ViewChangeStartMessage from nodeInBox queue, should be:
- move msgs from rxMsgs queue to nodeInBox queue
- process all 3PC msgs (for Prepare msg it should be moved to inBox queue of master_replica)
- add ViewChangeContinue msg into master_replica's inBox queue
- all not 3PC msgs will be stashed in strategy queue"""
looper.run(node.process_one_node_message(m))
m = node.master_replica.inBox.popleft()
assert isinstance(m[0], Prepare)
m = node.master_replica.inBox.popleft()
assert isinstance(m, ViewChangeContinueMessage)
m = node.view_changer.pre_vc_strategy.stashedNodeInBox.popleft()
assert isinstance(m[0], InstanceChange)
def test_get_msgs_from_rxMsgs_queue(create_node_and_not_start, looper):
node = create_node_and_not_start
node.view_changer = create_view_changer(node)
node.view_changer.pre_vc_strategy = VCStartMsgStrategy(view_changer, node)
node.view_changer.view_no = 0
"""pre_view_change stage"""
node.view_changer.start_view_change(1)
assert node.view_changer.view_no == 0
prepare = Prepare(
0,
0,
1,
get_utc_epoch(),
'f99937241d4c891c08e92a3cc25966607315ca66b51827b170d492962d58a9be',
'CZecK1m7VYjSNCC7pGHj938DSW2tfbqoJp1bMJEtFqvG',
'7WrAMboPTcMaQCU1raoj28vnhu2bPMMd2Lr9tEcsXeCJ')
inst_change = InstanceChange(1, 25)
m = node.nodeInBox.popleft()
assert isinstance(m[0], ViewChangeStartMessage)
node.nodestack.addRemote('someNode', genHa(), b'1DYuELN<SHbv1?NJ=][4De%^Hge887B0I!s<YGdD', 'pubkey')
node.nodestack.rxMsgs.append((json.dumps(prepare._asdict()), 'pubkey'))
node.nodestack.rxMsgs.append((json.dumps(inst_change._asdict()), 'pubkey'))
node.msgHasAcceptableViewNo = lambda *args, **kwargs: True
"""While processing ViewChangeStartMessage from nodeInBox queue, should be:
- move msgs from rxMsgs queue to nodeInBox queue
- process all 3PC msgs (for Prepare msg it should be moved to inBox queue of master_replica)
- add ViewChangeContinue msg into master_replica's inBox queue
- all not 3PC msgs will be stashed in strategy queue"""
looper.run(node.process_one_node_message(m))
m = node.master_replica.inBox.popleft()
assert isinstance(m[0], Prepare)
m = node.master_replica.inBox.popleft()
assert isinstance(m, ViewChangeContinueMessage)
m = node.view_changer.pre_vc_strategy.stashedNodeInBox.popleft()
assert isinstance(m[0], InstanceChange)
def createInstanceChangeMessage():
# Creating a message this way to exclude
# client-side validation of viewNo
goodViewNo = 1
badViewNo = "BAD"
icMsg = InstanceChange(viewNo=goodViewNo, reason=0)
icMsg._fields["viewNo"] = badViewNo
return icMsg
def test_the_same_order_as_in_NodeInBox_after_vc_continued(pre_vc_strategy):
replica = pre_vc_strategy.replica
pre_vc_strategy.view_changer.view_no = 1
m1 = (InstanceChange(3, 25), "Beta")
m2 = (InstanceChange(3, 25), "Gamma")
pre_vc_strategy.stashedNodeInBox.append(m1)
pre_vc_strategy.stashedNodeInBox.append(m2)
assert len(pre_vc_strategy.node.nodeInBox) == 0
pre_vc_strategy.on_view_change_continued(replica, ViewChangeContinueMessage(2))
assert len(pre_vc_strategy.node.nodeInBox) > 0
assert pre_vc_strategy.node.nodeInBox.popleft() == m1
assert pre_vc_strategy.node.nodeInBox.popleft() == m2
def test_stash_not_3PC_msgs(pre_vc_strategy, looper):
node = pre_vc_strategy.node
pre_vc_strategy.view_changer.view_no = 1
m1 = (InstanceChange(3, 25), "Beta")
m2 = (InstanceChange(3, 25), "Gamma")
assert len(pre_vc_strategy.stashedNodeInBox) == 0
node.nodeInBox.append(m1)
node.nodeInBox.append(m2)
looper.run(pre_vc_strategy.on_view_change_started(pre_vc_strategy.node,
ViewChangeStartMessage(2),
"some_node"))
assert len(pre_vc_strategy.stashedNodeInBox) > 0
"""In the same order as from nodeInBox queue"""
assert pre_vc_strategy.stashedNodeInBox.popleft() == m1
assert pre_vc_strategy.stashedNodeInBox.popleft() == m2
def _propose_view_change(self, suspision_code):
proposed_view_no = self._data.view_no + 1
self._logger.info(
"{} proposing a view change to {} with code {}".format(
self, proposed_view_no, suspision_code))
msg = InstanceChange(proposed_view_no, suspision_code)
self._network.send(msg)
def _send_instance_change(self, view_no: int, suspicion: Suspicion):
logger.info("{}{} sending an instance change with view_no {} since {}".
format(VIEW_CHANGE_PREFIX, self, view_no, suspicion.reason))
msg = InstanceChange(view_no, suspicion.code)
self._network.send(msg)
# record instance change vote for self and try to change the view if quorum is reached
self._on_verified_instance_change_msg(msg, self.name)
def test_instance_change_from_known(view_change_trigger_service):
current_view = view_change_trigger_service._data.view_no
proposed_view = current_view + 1
ic_msg = InstanceChange(viewNo=proposed_view, reason=26)
frm = list(view_change_trigger_service._data.validators)[0]
view_change_trigger_service.process_instance_change(ic_msg, frm=frm)
assert view_change_trigger_service._instance_changes.has_inst_chng_from(
proposed_view, frm)
def test_instance_change_from_unknown(view_change_trigger_service):
current_view = view_change_trigger_service._data.view_no
proposed_view = current_view + 1
ic_msg = InstanceChange(viewNo=proposed_view, reason=26)
frm = 'SomeUnknownNode'
view_change_trigger_service.process_instance_change(ic_msg, frm=frm)
assert not view_change_trigger_service._instance_changes.has_inst_chng_from(
proposed_view, frm)
def test_instance_changes_has_quorum_when_enough_distinct_votes_are_added(instance_changes):
quorum = 2
view_no = 1
assert not instance_changes.has_quorum(view_no, quorum)
for i in range(quorum):
instance_changes.add_vote(InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code),
"Node{}".format(i))
assert instance_changes.has_quorum(view_no, quorum)
def _propose_view_change(self, suspision_code):
proposed_view_no = self._data.view_no
# TODO: For some reason not incrementing view_no in most cases leads to lots of failing/flaky tests
# if suspicion == Suspicions.INSTANCE_CHANGE_TIMEOUT or not self.view_change_in_progress:
if suspision_code != Suspicions.STATE_SIGS_ARE_NOT_UPDATED or not self._data.waiting_for_new_view:
proposed_view_no += 1
self._logger.info(
"{} proposing a view change to {} with code {}".format(
self, proposed_view_no, suspision_code))
msg = InstanceChange(proposed_view_no, suspision_code)
self._network.send(msg)
def test_add_first_vote(instance_changes):
frm = "Node1"
view_no = 1
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
assert not instance_changes
instance_changes.add_vote(msg, frm)
assert instance_changes[view_no].msg == msg
assert instance_changes[view_no].voters[frm]
assert instance_changes.has_view(view_no)
assert instance_changes.has_inst_chng_from(view_no, frm)
def test_remove_view(instance_change_provider):
frm = "Node1"
view_no = 2
instance_change_provider.add_vote(
InstanceChange(view_no - 1, Suspicions.PRIMARY_DEGRADED.code), frm)
instance_change_provider.add_vote(
InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code), frm)
assert instance_change_provider.has_view(view_no - 1)
assert instance_change_provider.has_view(view_no)
assert instance_change_provider.has_inst_chng_from(view_no - 1, frm)
assert instance_change_provider.has_inst_chng_from(view_no, frm)
assert _is_view_in_db(view_no, instance_change_provider)
instance_change_provider.remove_view(view_no)
assert not instance_change_provider.has_view(view_no - 1)
assert not instance_change_provider.has_view(view_no)
assert not instance_change_provider.has_inst_chng_from(view_no - 1, frm)
assert not instance_change_provider.has_inst_chng_from(view_no, frm)
assert not _is_view_in_db(view_no, instance_change_provider)
def test_add_first_vote(instance_change_provider):
frm = "Node1"
view_no = 1
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
assert not instance_change_provider.has_view(view_no)
assert not instance_change_provider.has_inst_chng_from(view_no, frm)
instance_change_provider.add_vote(msg, frm)
assert instance_change_provider.has_view(view_no)
assert instance_change_provider.has_inst_chng_from(view_no, frm)
assert _is_view_in_db(view_no, instance_change_provider)
def test_add_vc_start_msg_during_start_view_change(txnPoolNodeSet,
looper):
delayed_node = txnPoolNodeSet[-1]
current_view_no = delayed_node.viewNo
proposed_view_no = current_view_no + 1
delayed_node.nodeIbStasher.delay(vcd_delay(1000))
delayed_node.nodeIbStasher.delay(icDelay(1000))
assert delayed_node.view_changer
ensure_view_change(looper, txnPoolNodeSet[:-1])
"""
If view number was incremented, that means that instanceChange quorum was archived
"""
assert txnPoolNodeSet[0].viewNo == proposed_view_no
looper.removeProdable(delayed_node)
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Alpha'))
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Beta'))
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Gamma'))
delayed_node.processNodeInBox = functools.partial(Node.processNodeInBox, delayed_node)
looper.run(delayed_node.processNodeInBox())
looper.run(delayed_node.serviceViewChanger(None))
assert len(delayed_node.nodeInBox) == 1
m = delayed_node.nodeInBox.popleft()
assert isinstance(m[0], ViewChangeStartMessage)
def test_equal_votes_dont_accumulate_when_added(instance_changes, tconf):
frm = "Node1"
view_no = 1
time_provider = MockTimestamp(0)
second_vote_time = 1
instance_changes = InstanceChanges(tconf, time_provider)
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_changes.add_vote(msg, frm)
time_provider.value = second_vote_time
instance_changes.add_vote(msg, frm)
assert instance_changes[view_no].voters[frm] == second_vote_time
assert len(instance_changes[view_no].voters) == 1
assert len(instance_changes) == 1
def test_old_ic_discard(instance_change_provider, tconf, time_provider):
frm = "Node1"
view_no = 1
quorum = 2
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
time_provider.value = 0
instance_change_provider.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_change_provider.has_view(view_no)
instance_change_provider.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_change_provider.has_inst_chng_from(view_no, frm)
instance_change_provider.add_vote(msg, frm)
time_provider.value += tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1
assert not instance_change_provider.has_quorum(view_no, quorum)
def test_equal_votes_dont_accumulate_when_added(instance_change_provider,
time_provider):
frm = "Node1"
view_no = 1
quorum = 2
time_provider.value = 0
second_vote_time = 1
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_change_provider.add_vote(msg, frm)
time_provider.value = second_vote_time
instance_change_provider.add_vote(msg, frm)
assert instance_change_provider.has_view(view_no)
assert instance_change_provider.has_inst_chng_from(view_no, frm)
assert not instance_change_provider.has_quorum(view_no, quorum)
instance_changes_db = instance_change_provider._node_status_db.get(
instance_change_provider.generate_db_key(view_no))
assert len(node_status_db_serializer.deserialize(instance_changes_db)) == 1
def test_too_old_messages_dont_count_towards_quorum(instance_changes, tconf):
frm1 = "Node1"
frm2 = "Node2"
view_no = 1
quorum = 2
time_provider = MockTimestamp(0)
instance_changes = InstanceChanges(tconf, time_provider)
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_changes.add_vote(msg, frm1)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2)
instance_changes.add_vote(msg, frm2)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL/2) + 1
assert not instance_changes.has_quorum(view_no, quorum)
assert instance_changes.has_view(view_no)
assert instance_changes[view_no].msg == msg
assert not instance_changes.has_inst_chng_from(view_no, frm1)
assert instance_changes.has_inst_chng_from(view_no, frm2)
def test_too_old_messages_dont_count_towards_quorum(instance_change_provider,
time_provider, tconf):
frm1 = "Node1"
frm2 = "Node2"
view_no = 1
quorum = 2
time_provider.value = 0
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
instance_change_provider.add_vote(msg, frm1)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL / 2)
instance_change_provider.add_vote(msg, frm2)
assert instance_change_provider.has_quorum(view_no, quorum)
time_provider.value += (tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL /
2) + 1
assert not instance_change_provider.has_quorum(view_no, quorum)
assert not instance_change_provider.has_inst_chng_from(view_no, frm1)
assert instance_change_provider.has_inst_chng_from(view_no, frm2)
instance_changes_db = instance_change_provider._node_status_db.get(
instance_change_provider.generate_db_key(view_no))
assert len(node_status_db_serializer.deserialize(instance_changes_db)) == 1
def test_update_instance_changes_in_db(instance_change_provider, tconf,
node_status_db, time_provider):
frm = "Node1"
view_no = 1
msg = InstanceChange(view_no, Suspicions.PRIMARY_DEGRADED.code)
assert not instance_change_provider.has_view(view_no)
assert not instance_change_provider.has_inst_chng_from(view_no, frm)
assert not _is_view_in_db(view_no, instance_change_provider)
instance_change_provider.add_vote(msg, frm)
assert instance_change_provider.has_view(view_no)
assert instance_change_provider.has_inst_chng_from(view_no, frm)
assert _is_view_in_db(view_no, instance_change_provider)
instance_change_provider._node_status_db.close()
assert instance_change_provider._node_status_db.closed
instance_change_provider._node_status_db.open()
new_instance_change_provider = InstanceChangeProvider(
tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL, node_status_db,
time_provider)
assert new_instance_change_provider.has_view(view_no)
assert new_instance_change_provider.has_inst_chng_from(view_no, frm)
def _create_instance_change_msg(self, view_no, suspicion_code):
return InstanceChange(view_no, suspicion_code)