def test_process_backup_instance_faulty_msg_quorum_for_different_replicas(backup_instance_faulty_processor):
'''
Test for case when process_backup_instance_faulty_msg received messages for different instances.
Firstly f messages for [1,2], secondly 1 message for [2]. As a result, replica 2 should be removed
and replica 1 are not.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED = "quorum"
instance_to_remove = 2
instance_not_removed = 1
nodes = set()
msg = BackupInstanceFaulty(node.viewNo,
[instance_to_remove, instance_not_removed],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
# send node.quorums.backup_instance_faulty - 1 BackupInstanceFaulty messages for 1, 2 replicas
for node_name in node.allNodeNames[:node.quorums.backup_instance_faulty.value - 1]:
nodes.add(node_name)
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node_name)
assert not node.replicas.remove_replica_calls
msg = BackupInstanceFaulty(node.viewNo,
[instance_to_remove],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
# send BackupInstanceFaulty message for 2 replica
node_name = node.allNodeNames[node.quorums.backup_instance_faulty.value - 1]
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node_name)
# check that 2nd replica was removed and 1st replica did not.
assert instance_to_remove not in backup_instance_faulty_processor.backup_instances_faulty
assert len(node.replicas.remove_replica_calls) == 1
assert node.replicas.remove_replica_calls[0] == instance_to_remove
assert nodes.issubset(backup_instance_faulty_processor.backup_instances_faulty[instance_not_removed].keys())
def test_process_backup_instance_faulty_without_quorum(
backup_instance_faulty_processor):
'''
Send (quorum - 1) messages from different nodes and (quorum - 1) from current node.
Replica shouldn't be removed in both cases.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_DEGRADATION = "quorum"
instance_to_remove = 1
msg = BackupInstanceFaulty(node.viewNo, [instance_to_remove],
Suspicions.BACKUP_PRIMARY_DEGRADED.code)
nodes = set()
# check that node.quorums.backup_instance_faulty - 1 messages don't leads to replica removing
for node_name in node.allNodeNames[1:node.quorums.backup_instance_faulty.
value - 1]:
nodes.add(node_name)
backup_instance_faulty_processor.process_backup_instance_faulty_msg(
msg, node_name)
assert nodes.issubset(backup_instance_faulty_processor.
backup_instances_faulty[instance_to_remove].keys())
assert not node.replicas.remove_replica_calls
# check that node.quorums.backup_instance_faulty - 1 own messages don't lead to replica removing
for _ in range(node.quorums.backup_instance_faulty.value - 1):
backup_instance_faulty_processor.process_backup_instance_faulty_msg(
msg, node.name)
assert nodes.issubset(backup_instance_faulty_processor.
backup_instances_faulty[instance_to_remove].keys())
assert backup_instance_faulty_processor.backup_instances_faulty[instance_to_remove][node.name] == \
node.quorums.backup_instance_faulty.value - 1
assert not node.replicas.remove_replica_calls
def test_process_backup_instance_faulty_msg_quorum_degradation(
backup_instance_faulty_processor):
'''
Send (quorum - 1) of messages with BACKUP_PRIMARY_DEGRADED from different nodes and
check that it is not lead to a replica removing. But after sending one more message,
the replica will be removed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_DEGRADATION = "quorum"
instance_to_remove = 1
msg = BackupInstanceFaulty(node.viewNo, [instance_to_remove],
Suspicions.BACKUP_PRIMARY_DEGRADED.code)
nodes = set()
# check that node.quorums.backup_instance_faulty - 1 messages don't leads to replica removing
for node_name in node.allNodeNames[:node.quorums.backup_instance_faulty.
value - 1]:
nodes.add(node_name)
backup_instance_faulty_processor.process_backup_instance_faulty_msg(
msg, node_name)
assert nodes.issubset(backup_instance_faulty_processor.
backup_instances_faulty[instance_to_remove])
assert not node.replicas.remove_replica_calls
# check that messages from all nodes lead to replica removing
for node_name in node.allNodeNames[node.quorums.backup_instance_faulty.
value - 1:]:
backup_instance_faulty_processor.process_backup_instance_faulty_msg(
msg, node_name)
assert not backup_instance_faulty_processor.backup_instances_faulty
assert len(node.replicas.remove_replica_calls) == 1
assert node.replicas.remove_replica_calls[0] == instance_to_remove
def test_quorum_collection(tdir):
node = FakeNode(tdir)
proc = BackupInstanceFaultyProcessor(node)
backup_faulty = BackupInstanceFaulty(0, [1], Suspicions.BACKUP_PRIMARY_DEGRADED.code)
proc.process_backup_instance_faulty_msg(backup_faulty, 'someone')
assert len(proc.backup_instances_faulty[1].keys()) == 1
def __send_backup_instance_faulty(self, instances: List[int],
reason: Suspicion):
if not self.node.view_change_in_progress and not instances:
return
logger.info(
"{} sending a backup instance faulty message with view_no {} "
"and reason '{}' for instances: {}".format(self.node.name,
self.node.viewNo,
reason.reason,
instances))
msg = BackupInstanceFaulty(self.node.viewNo, instances, reason.code)
self.process_backup_instance_faulty_msg(msg, self.node.name)
self.node.send(msg)
def test_process_backup_instance_empty_msg(backup_instance_faulty_processor):
'''
Check that BackupInstanceFaulty message with empty list of instances will not be processed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_DEGRADATION = "quorum"
msg = BackupInstanceFaulty(node.viewNo, [],
Suspicions.BACKUP_PRIMARY_DEGRADED.code)
backup_instance_faulty_processor.process_backup_instance_faulty_msg(
msg, node.name)
assert not backup_instance_faulty_processor.backup_instances_faulty
assert not node.replicas.remove_replica_calls
def test_process_backup_instance_faulty_msg_local_disconnection(backup_instance_faulty_processor):
'''
Check that BackupInstanceFaulty message with REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED
will not be processed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED = "local"
msg = BackupInstanceFaulty(node.viewNo,
[1, 2],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node.name)
assert not backup_instance_faulty_processor.backup_instances_faulty
assert not node.replicas.remove_replica_calls
def __send_backup_instance_faulty(self, instances: List[int],
reason: Suspicion):
if not self.node.view_change_in_progress and instances and \
not any(inst_id in self.backup_instances_faulty and
self.node.name in self.backup_instances_faulty[inst_id]
for inst_id in instances):
return
logger.info("{} sending an backup instance faulty with view_no {} "
"and reason {} for instances: ".format(
self.node.name, self.node.viewNo, reason.reason,
instances))
for inst_id in instances:
self.backup_instances_faulty.setdefault(inst_id, set()) \
.add(self.node.name)
self.node.send(
BackupInstanceFaulty(self.node.viewNo, instances, reason.code))
def test_process_backup_instance_faulty_msg_quorum_from_itself(backup_instance_faulty_processor):
'''
Send quorum of messages from current node.
Replica should be removed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED = "quorum"
instance_to_remove = 1
msg = BackupInstanceFaulty(node.viewNo,
[instance_to_remove],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
# check that node.quorums.backup_instance_faulty own messages lead to replica removing
for _ in range(node.quorums.backup_instance_faulty.value):
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node.name)
assert not backup_instance_faulty_processor.backup_instances_faulty
assert len(node.replicas.remove_replica_calls) == 1
assert node.replicas.remove_replica_calls[0] == instance_to_remove
def test_process_backup_instance_faulty_msg_quorum_disconnection(backup_instance_faulty_processor):
'''
Send quorum of messages with BACKUP_PRIMARY_DISCONNECTED from different nodes.
Replica should be removed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED = "quorum"
instance_to_remove = 1
msg = BackupInstanceFaulty(node.viewNo,
[instance_to_remove],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
for node_name in node.allNodeNames:
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node_name)
# check that messages from all nodes lead to replica removing
assert not backup_instance_faulty_processor.backup_instances_faulty
assert len(node.replicas.remove_replica_calls) == 1
assert node.replicas.remove_replica_calls[0] == instance_to_remove
def test_process_backup_instance_faulty_msg_quorum_from_others(backup_instance_faulty_processor):
'''
Send quorum of messages from different nodes exception current node.
Replica should be removed.
'''
node = backup_instance_faulty_processor.node
node.config.REPLICAS_REMOVING_WITH_PRIMARY_DISCONNECTED = "quorum"
instance_to_remove = 1
msg = BackupInstanceFaulty(node.viewNo,
[instance_to_remove],
Suspicions.BACKUP_PRIMARY_DISCONNECTED.code)
# send messages from all nodes with exception of current node
for node_name in node.allNodeNames[1:]:
backup_instance_faulty_processor.process_backup_instance_faulty_msg(msg, node_name)
# check that remove_replica was called
assert not backup_instance_faulty_processor.backup_instances_faulty
assert len(node.replicas.remove_replica_calls) == 1
assert node.replicas.remove_replica_calls[0] == instance_to_remove
