def checkPrimaryPlacement():
        # Node names sorted by rank
        sortedNodeNames = sorted(nodeSet.nodes.values(),
                          key=operator.attrgetter("rank"))

        for idx, node in enumerate(sortedNodeNames):
            # For instance 0, the primary replica should be on the node with rank 0
            if idx == 0:
                primaryName = Replica.generateName(sortedNodeNames[idx], 0)
                assert node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 1, the primary replica should be on the node with rank 1
            if idx == 1:
                primaryName = Replica.generateName(sortedNodeNames[idx], 1)
                assert not node.replicas[0].isPrimary
                assert node.replicas[1].isPrimary
                assert not node.replicas[2].isPrimary

            # For instance 2, the primary replica should be on the node with rank 2
            if idx == 2:
                primaryName = Replica.generateName(sortedNodeNames[idx], 2)
                assert not node.replicas[0].isPrimary
                assert not node.replicas[1].isPrimary
                assert node.replicas[2].isPrimary
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
    """
    Case 2 - A node making nominations for a multiple other nodes. Consider 4
    nodes A, B, C, and D. Lets say node B is malicious and nominates node C
    to all nodes. Again node B nominates node D to all nodes.
    """
    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    looper.run(checkNodesConnected(nodeSet))

    # Node B sends multiple NOMINATE msgs but only after A has nominated itself
    looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))

    instId = getSelfNominationByNode(A)

    BRep = Replica.generateName(B.name, instId)
    CRep = Replica.generateName(C.name, instId)
    DRep = Replica.generateName(D.name, instId)

    # Node B first sends NOMINATE msgs for Node C to all nodes
    B.send(Nomination(CRep, instId, B.viewNo))
    # Node B sends NOMINATE msgs for Node D to all nodes
    B.send(Nomination(DRep, instId, B.viewNo))

    # Ensure elections are done
    ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)

    # All nodes from node A, node C, node D(node B is malicious anyway so
    # not considering it) should have nomination for node C from node B since
    #  node B first nominated node C
    for node in [A, C, D]:
        assert node.elector.nominations[instId][BRep] == CRep
Exemple #3
0
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
    """
    Case 2 - A node making nominations for a multiple other nodes. Consider 4
    nodes A, B, C, and D. Lets say node B is malicious and nominates node C
    to all nodes. Again node B nominates node D to all nodes.
    """
    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    looper.run(checkNodesConnected(nodeSet))

    # Node B sends multiple NOMINATE msgs but only after A has nominated itself
    looper.run(eventually(checkNomination, A, A.name, retryWait=.25,
                          timeout=1))

    instId = getSelfNominationByNode(A)

    BRep = Replica.generateName(B.name, instId)
    CRep = Replica.generateName(C.name, instId)
    DRep = Replica.generateName(D.name, instId)

    # Node B first sends NOMINATE msgs for Node C to all nodes
    B.send(Nomination(CRep, instId, B.viewNo))
    # Node B sends NOMINATE msgs for Node D to all nodes
    B.send(Nomination(DRep, instId, B.viewNo))

    # Ensure elections are done
    ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)

    # All nodes from node A, node C, node D(node B is malicious anyway so
    # not considering it) should have nomination for node C from node B since
    #  node B first nominated node C
    for node in [A, C, D]:
        assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes):
    """
    Case 5 - A node making primary declarations for a multiple other nodes.
    Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
    declares node C as primary to all nodes.
    Again node B declares node D as primary to all nodes.
    """
    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    looper.run(checkNodesConnected(nodeSet))

    BRep = Replica.generateName(B.name, 0)
    CRep = Replica.generateName(C.name, 0)
    DRep = Replica.generateName(D.name, 0)

    # Node B first sends PRIMARY msgs for Node C to all nodes
    B.send(Primary(CRep, 0, B.viewNo))
    # Node B sends PRIMARY msgs for Node D to all nodes
    B.send(Primary(DRep, 0, B.viewNo))

    # Ensure elections are done
    ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)

    # All nodes from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have primary declarations for node C from node B
    #  since node B first nominated node C
    for node in [A, C, D]:
        logger.debug("node {} should have primary declaration for C from node B".format(node))
        assert node.elector.primaryDeclarations[0][BRep] == CRep
Exemple #5
0
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes):
    """
    Case 5 - A node making primary declarations for a multiple other nodes.
    Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
    declares node C as primary to all nodes.
    Again node B declares node D as primary to all nodes.
    """
    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    looper.run(checkNodesConnected(nodeSet))

    BRep = Replica.generateName(B.name, 0)
    CRep = Replica.generateName(C.name, 0)
    DRep = Replica.generateName(D.name, 0)

    # Node B first sends PRIMARY msgs for Node C to all nodes
    B.send(Primary(CRep, 0, B.viewNo))
    # Node B sends PRIMARY msgs for Node D to all nodes
    B.send(Primary(DRep, 0, B.viewNo))

    # Ensure elections are done
    ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)

    # All nodes from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have primary declarations for node C from node B
    #  since node B first nominated node C
    for node in [A, C, D]:
        logger.debug(
            "node {} should have primary declaration for C from node B"
            .format(node))
        assert node.elector.primaryDeclarations[0][BRep] == CRep
 def checkPresence():
     for node in [C, D]:
         commReqs = getCommits(node, 0)
         assert len(commReqs) > 0
         assert Replica.generateName(A.name, 0) not in commReqs[0][1]
         commReqs = getCommits(node, 1)
         assert len(commReqs) > 0
         assert Replica.generateName(A.name, 1) in commReqs[0][1]
 def checkPresence():
     for node in [C, D]:
         commReqs = getCommits(node, 0)
         assert len(commReqs) > 0
         assert Replica.generateName(A.name, 0) not in commReqs[0][0]
         commReqs = getCommits(node, 1)
         assert len(commReqs) > 0
         assert Replica.generateName(A.name, 1) in commReqs[0][0]
Exemple #8
0
def checkNomination(node: TestNode, nomineeName: str):
    matches = [replica.name for instId, replica in node.elector.replicas.items() if
               node.elector.didReplicaNominate(instId) is True and
               replica.name in node.elector.nominations[instId] and
               node.elector.nominations[instId][replica.name][0] ==
               Replica.generateName(nomineeName, instId)]
    assert len(matches) > 0
    return matches[0]
 def next_primary_replica_name(self, instance_id):
     """
     Returns name of the next node which is supposed to be a new Primary
     in round-robin fashion
     """
     return Replica.generateName(
         nodeName=self.next_primary_node_name(instance_id),
         instId=instance_id)
Exemple #10
0
def checkNomination(node: TestNode, nomineeName: str):
    matches = [replica.name for instId, replica in enumerate(node.elector.replicas) if
               node.elector.didReplicaNominate(instId) is True and
               replica.name in node.elector.nominations[instId] and
               node.elector.nominations[instId][replica.name][0] ==
               Replica.generateName(nomineeName, instId)]
    assert len(matches) > 0
    return matches[0]
Exemple #11
0
def getSelfNominationByNode(node: TestNode) -> int:
    """
    This function returns the index of the protocol instance for which it nominated itself
    @param node: the node
    @return: the protocol instance index
    """
    for instId, replica in enumerate(node.elector.replicas):
        name = Replica.generateName(node.name, instId)
        if node.elector.nominations.get(instId, {}).get(name, None) == name:
            return instId
Exemple #12
0
def getSelfNominationByNode(node: TestNode) -> int:
    """
    This function returns the index of the protocol instance for which it nominated itself
    @param node: the node
    @return: the protocol instance index
    """
    for instId, replica in enumerate(node.elector.replicas):
        name = Replica.generateName(node.name, instId)
        if node.elector.nominations.get(instId, {}).get(name, None) == name:
            return instId
 def next_primary_replica_name_for_master(self, nodeReg=None):
     """
     Returns name and corresponding instance name of the next node which
     is supposed to be a new Primary. In fact it is not round-robin on
     this abstraction layer as currently the primary of master instance is
     pointed directly depending on view number, instance id and total
     number of nodes.
     But since the view number is incremented by 1 before primary selection
     then current approach may be treated as round robin.
     """
     name = self._next_primary_node_name_for_master(nodeReg)
     return name, Replica.generateName(nodeName=name, instId=0)
 def next_primary_replica_name_for_master(self, nodeReg=None):
     """
     Returns name and corresponding instance name of the next node which
     is supposed to be a new Primary. In fact it is not round-robin on
     this abstraction layer as currently the primary of master instance is
     pointed directly depending on view number, instance id and total
     number of nodes.
     But since the view number is incremented by 1 before primary selection
     then current approach may be treated as round robin.
     """
     name = self._next_primary_node_name_for_master(nodeReg)
     return name, Replica.generateName(nodeName=name, instId=0)
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
    """
    Case 1 - A node making multiple nominations for a particular node. Consider
    4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
    nominating Node D
    """
    nodes = keySharedNodes
    nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]

    # Doesn't matter if nodes reach the ready state or not. Just start them
    looper.run(checkNodesConnected(nodes))

    # Node B sends multiple NOMINATE messages for Node D but only after A has
    # nominated itself
    timeout = waits.expectedPoolNominationTimeout(
        nodeCount=len(keySharedNodes))
    looper.run(
        eventually(checkNomination,
                   nodeA,
                   nodeA.name,
                   retryWait=.25,
                   timeout=timeout))

    instId = getSelfNominationByNode(nodeA)

    for i in range(5):
        # nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
        nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
    nodeB.nodestack.flushOutBoxes()

    # No node from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have more than one nomination for node D since
    # node D is slow. The one nomination for D, that nodes A, C
    # and D might have would be because of node B
    for node in [nodeA, nodeC, nodeD]:
        assert [n[0] for n in node.elector.nominations[instId].values()].count(
            Replica.generateName(nodeD.name, instId)) \
            <= 1

    timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
    primaryReplicas = ensureElectionsDone(looper=looper,
                                          nodes=nodes,
                                          customTimeout=timeout)

    for node in nodes:
        logger.debug("{}'s nominations {}".format(node,
                                                  node.elector.nominations))
    # Node D should not have any primary
    assert not nodeD.hasPrimary
    # A node other than Node D should not have any replica among the
    # primary replicas
    assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet):
    """
    Case 5 - A node making primary declarations for a multiple other nodes.
    Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
    declares node C as primary to all nodes.
    Again node B declares node D as primary to all nodes.
    """
    A, B, C, D = txnPoolNodeSet

    looper.run(checkNodesConnected(txnPoolNodeSet))

    BRep = Replica.generateName(B.name, 0)
    CRep = Replica.generateName(C.name, 0)
    DRep = Replica.generateName(D.name, 0)

    # Node B first sends PRIMARY msgs for Node C to all nodes
    # B.send(Primary(CRep, 0, B.viewNo))
    B.send(primaryByNode(CRep, B, 0))
    # Node B sends PRIMARY msgs for Node D to all nodes
    # B.send(Primary(DRep, 0, B.viewNo))
    B.send(primaryByNode(DRep, B, 0))

    # Ensure elections are done
    # also have to take into account the catchup procedure
    timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
              waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \
              delayOfElectionDone
    ensureElectionsDone(looper=looper,
                        nodes=txnPoolNodeSet,
                        customTimeout=timeout)

    # All nodes from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have primary declarations for node C from node B
    #  since node B first nominated node C
    for node in [A, C, D]:
        logger.debug(
            "node {} should have primary declaration for C from node B".format(
                node))
        assert node.elector.primaryDeclarations[0][BRep][0] == CRep
def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet):
    """
    Case 5 - A node making primary declarations for a multiple other nodes.
    Consider 4 nodes A, B, C, and D. Lets say node B is malicious and
    declares node C as primary to all nodes.
    Again node B declares node D as primary to all nodes.
    """
    A, B, C, D = txnPoolNodeSet

    looper.run(checkNodesConnected(txnPoolNodeSet))

    BRep = Replica.generateName(B.name, 0)
    CRep = Replica.generateName(C.name, 0)
    DRep = Replica.generateName(D.name, 0)

    # Node B first sends PRIMARY msgs for Node C to all nodes
    # B.send(Primary(CRep, 0, B.viewNo))
    B.send(primaryByNode(CRep, B, 0))
    # Node B sends PRIMARY msgs for Node D to all nodes
    # B.send(Primary(DRep, 0, B.viewNo))
    B.send(primaryByNode(DRep, B, 0))

    # Ensure elections are done
    # also have to take into account the catchup procedure
    timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
              waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \
              delayOfElectionDone
    ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=timeout)

    # All nodes from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have primary declarations for node C from node B
    #  since node B first nominated node C
    for node in [A, C, D]:
        logger.debug(
            "node {} should have primary declaration for C from node B"
                .format(node))
        assert node.elector.primaryDeclarations[0][BRep][0] == CRep
Exemple #18
0
 def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
                                          primaries, node_reg, node_ids):
     """
     Returns name and corresponding instance name of the next node which
     is supposed to be a new Primary for backup instance in round-robin
     fashion starting from primary of master instance.
     """
     if node_reg is None:
         node_reg = self.node.nodeReg
     total_nodes = len(node_reg)
     rank = (master_primary_rank + 1) % total_nodes
     name = self.node.get_name_by_rank(rank, node_reg, node_ids)
     while name in primaries:
         rank = (rank + 1) % total_nodes
         name = self.node.get_name_by_rank(rank, node_reg, node_ids)
     return name, Replica.generateName(nodeName=name, instId=instance_id)
 def next_primary_replica_name_for_backup(self, instance_id, master_primary_rank,
                                          primaries, nodeReg=None):
     """
     Returns name and corresponding instance name of the next node which
     is supposed to be a new Primary for backup instance in round-robin
     fashion starting from primary of master instance.
     """
     if nodeReg is None:
         nodeReg = self.node.nodeReg
     total_nodes = len(nodeReg)
     rank = (master_primary_rank + 1) % total_nodes
     name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
     while name in primaries:
         rank = (rank + 1) % total_nodes
         name = self.node.get_name_by_rank(rank, nodeReg=nodeReg)
     return name, Replica.generateName(nodeName=name, instId=instance_id)
def testPrimaryElectionCase1(case1Setup, looper, txnPoolNodeSet):
    """
    Case 1 - A node making multiple nominations for a particular node. Consider
    4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
    nominating Node D
    """
    nodeA, nodeB, nodeC, nodeD = txnPoolNodeSet

    # Doesn't matter if nodes reach the ready state or not. Just start them
    looper.run(checkNodesConnected(txnPoolNodeSet))

    # Node B sends multiple NOMINATE messages for Node D but only after A has
    # nominated itself
    timeout = waits.expectedPoolNominationTimeout(
        nodeCount=len(txnPoolNodeSet))
    looper.run(eventually(checkNomination, nodeA, nodeA.name,
                          retryWait=.25, timeout=timeout))

    instId = getSelfNominationByNode(nodeA)

    for i in range(5):
        # nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
        nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
    nodeB.nodestack.flushOutBoxes()

    # No node from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have more than one nomination for node D since
    # node D is slow. The one nomination for D, that nodes A, C
    # and D might have would be because of node B
    for node in [nodeA, nodeC, nodeD]:
        assert [n[0] for n in node.elector.nominations[instId].values()].count(
            Replica.generateName(nodeD.name, instId)) \
               <= 1

    timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
    primaryReplicas = ensureElectionsDone(looper=looper,
                                          nodes=txnPoolNodeSet, customTimeout=timeout)

    for node in txnPoolNodeSet:
        logger.debug(
            "{}'s nominations {}".format(node, node.elector.nominations))
    # Node D should not have any primary
    assert not nodeD.hasPrimary
    # A node other than Node D should not have any replica among the
    # primary replicas
    assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
    """
    Case 1 - A node making multiple nominations for a particular node. Consider
    4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
    nominating Node D
    """
    nodes = keySharedNodes
    nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]

    # Doesn't matter if nodes reach the ready state or not. Just start them
    looper.run(checkNodesConnected(nodes))

    # Node B sends multiple NOMINATE msgs for Node D but only after A has
    # nominated itself
    looper.run(eventually(checkNomination, nodeA, nodeA.name, retryWait=.25,
                          timeout=1))

    instId = getSelfNominationByNode(nodeA)

    for i in range(5):
        nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
    nodeB.flushOutBoxes()

    # No node from node A, node C, node D(node B is malicious anyway so not
    # considering it) should have more than one nomination for node D since
    # node D is slow. The one nomination for D, that nodes A, C
    # and D might have would be because of node B
    for node in [nodeA, nodeC, nodeD]:
        assert list(node.elector.nominations[instId].values()).count(
            Replica.generateName(nodeD.name, instId)) \
               <= 1

    primaryReplicas = ensureElectionsDone(looper=looper, nodes=nodes,
                                          retryWait=1, timeout=30)

    for node in nodes:
        logging.debug(
            "{}'s nominations {}".format(node, node.elector.nominations))
    # Node D should not have any primary
    assert not nodeD.hasPrimary
    # A node other than Node D should not have any replica among the
    # primary replicas
    assert nodeD.name not in [pr.name for pr in primaryReplicas]
Exemple #22
0
    def on_catchup_complete(self):
        # Select primaries after usual catchup (not view change)
        ledger = self.node.getLedger(AUDIT_LEDGER_ID)
        self.node.backup_instance_faulty_processor.restore_replicas()
        self.node.drop_primaries()
        if len(ledger) == 0:
            self.node.select_primaries()
        else:
            # Emulate view change start
            self.node.view_changer.previous_view_no = self.node.viewNo
            self.node.viewNo = get_payload_data(ledger.get_last_committed_txn())[AUDIT_TXN_VIEW_NO]
            self.node.view_changer.previous_master_primary = self.node.master_primary_name
            self.node.view_changer.set_defaults()

            self.node.primaries = self._get_last_audited_primaries()
            if len(self.replicas) != len(self.node.primaries):
                logger.error('Audit ledger has inconsistent number of nodes. '
                             'Node primaries = {}'.format(self.node.primaries))
            if any(p not in self.node.nodeReg for p in self.node.primaries):
                logger.error('Audit ledger has inconsistent names of primaries. '
                             'Node primaries = {}'.format(self.node.primaries))
            # Similar functionality to select_primaries
            for instance_id, replica in self.replicas.items():
                if instance_id == 0:
                    self.node.start_participating()
                replica.primaryChanged(
                    Replica.generateName(self.node.primaries[instance_id], instance_id))
                self.node.primary_selected(instance_id)

        # Primary propagation
        last_sent_pp_seq_no_restored = False
        for replica in self.replicas.values():
            replica.on_propagate_primary_done()
        if self.node.view_changer.previous_view_no == 0:
            last_sent_pp_seq_no_restored = \
                self.node.last_sent_pp_store_helper.try_restore_last_sent_pp_seq_no()
        if not last_sent_pp_seq_no_restored:
            self.node.last_sent_pp_store_helper.erase_last_sent_pp_seq_no()

        # Emulate view_change ending
        self.node.on_view_propagated()