Example #1
0
def test_node_load_after_one_node_drops_all_msgs(looper, txnPoolNodeSet, tconf,
                                                 tdirWithPoolTxns,
                                                 allPluginsPath,
                                                 poolTxnStewardData, capsys):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())

    nodes = txnPoolNodeSet
    x = nodes[-1]

    with capsys.disabled():
        print("Patching node {}".format(x))

    def handleOneNodeMsg(self, wrappedMsg):
        # do nothing with an incoming node message
        pass

    x.handleOneNodeMsg = MethodType(handleOneNodeMsg, x)

    client_batches = 120
    txns_per_batch = 25
    for i in range(client_batches):
        s = perf_counter()
        sendReqsToNodesAndVerifySuffReplies(looper,
                                            wallet,
                                            client,
                                            txns_per_batch,
                                            override_timeout_limit=True)
        with capsys.disabled():
            print('{} executed {} client txns in {:.2f} seconds'.format(
                i + 1, txns_per_batch,
                perf_counter() - s))
Example #2
0
def nodeCreatedAfterSomeTxns(txnPoolNodeSet, tdirWithPoolTxns,
                             poolTxnStewardData, tconf, allPluginsPath,
                             request):
    with Looper(debug=True) as looper:
        client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                                  tdirWithPoolTxns,
                                                  clientClass=TestClient)
        looper.add(client)
        looper.run(client.ensureConnectedToNodes())
        txnCount = getValueFromModule(request, "txnCount", 5)
        sendReqsToNodesAndVerifySuffReplies(looper,
                                            wallet,
                                            client,
                                            txnCount,
                                            timeoutPerReq=25)

        newStewardName = randomString()
        newNodeName = "Epsilon"
        newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
            looper,
            client,
            wallet,
            newStewardName,
            newNodeName,
            tdirWithPoolTxns,
            tconf,
            allPluginsPath=allPluginsPath,
            autoStart=True)
        yield looper, newNode, client, wallet, newStewardClient, \
              newStewardWallet
Example #3
0
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
                                        tdirWithPoolTxns, poolTxnClientData):
    """
    Client should not be able to connect to nodes even after it has changed
    its HA. Since running on a local environment, only checking change of port.
    Dont know how to change IP.
    :return:
    """
    # TODO: Check for change of IP too
    # name, seed = poolTxnClientData
    # signer = SimpleSigner(seed=seed)
    # name = "testClient96541"
    # ha = genHa()
    # client = genTestClient(txnPoolNodeSet, signer=signer, ha=ha,
    #                        tmpdir=tdirWithPoolTxns, usePoolLedger=True,
    #                        name=name)
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
    basedirpath = client.basedirpath
    looper.removeProdable(client)

    # Removing RAET keep directory otherwise the client will use the same port
    #  since it will a directory of its name in the keep
    shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True)

    ha = genHa()
    client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId,
                              ha=ha, tmpdir=tdirWithPoolTxns,
                              usePoolLedger=True, name=client.name)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
Example #4
0
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
                                        tdirWithPoolTxns, poolTxnClientData):
    """
    Client should not be able to connect to nodes even after it has changed
    its HA. Since running on a local environment, only checking change of port.
    Dont know how to change IP.
    :return:
    """
    # TODO: Check for change of IP too
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
    basedirpath = client.basedirpath
    looper.removeProdable(client)

    # Removing RAET keep directory otherwise the client will use the same port
    #  since it will a directory of its name in the keep
    shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True)

    ha = genHa()
    client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId,
                              ha=ha, tmpdir=tdirWithPoolTxns,
                              usePoolLedger=True, name=client.name)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
Example #5
0
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
                     tdirWithPoolTxns, poolTxnData, noRetryReq):
    """
    Client should not be using node registry but pool transaction file
    :return:
    """
    clients = []
    for name in poolTxnClientNames:
        seed = poolTxnData["seeds"][name].encode()
        client, wallet = buildPoolClientAndWallet((name, seed),
                                                  tdirWithPoolTxns)
        txnPoolNodesLooper.add(client)
        ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client,
                                                      *txnPoolNodeSet)
        clients.append((client, wallet))

    n = 250
    timeOutPerReq = 3
    for (client, wallet) in clients:
        logger.debug("{} sending {} requests".format(client, n))
        sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
                                            n, 1, timeOutPerReq)
        logger.debug("{} sent {} requests".format(client, n))
    for node in txnPoolNodeSet:
        logger.debug("{} has requests {} with size {}".
                     format(node, len(node.requests), get_size(node.requests)))
        for replica in node.replicas:
            logger.debug("{} has prepares {} with size {}".
                         format(replica, len(replica.prepares),
                                get_size(replica.prepares)))
            logger.debug("{} has commits {} with size {}".
                         format(replica, len(replica.commits),
                                get_size(replica.commits)))
Example #6
0
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
                                        tdirWithPoolTxns,
                                        tdirWithClientPoolTxns,
                                        poolTxnClientData):
    """
    Client should not be able to connect to nodes even after it has changed
    its HA. Since running on a local environment, only checking change of port.
    Dont know how to change IP.
    :return:
    """
    # TODO: Check for change of IP too
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithClientPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
    keys_dir = os.path.join(client.keys_dir, client.name)
    client.stop()
    looper.removeProdable(client)

    ha = genHa()
    client, _ = genTestClient(txnPoolNodeSet,
                              identifier=wallet.defaultId,
                              ha=ha,
                              tmpdir=tdirWithClientPoolTxns,
                              usePoolLedger=True,
                              name=client.name)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *txnPoolNodeSet)
Example #7
0
def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation,
                             txnPoolNodeSet, tdir, tdirWithClientPoolTxns,
                             poolTxnStewardData, tconf, allPluginsPath,
                             request):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithClientPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    txnCount = getValueFromModule(request, "txnCount", 5)
    sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txnCount)
    newStewardName = randomString()
    newNodeName = "Epsilon"
    newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
        looper,
        client,
        wallet,
        newStewardName,
        newNodeName,
        tdir,
        tdirWithClientPoolTxns,
        tconf,
        nodeClass=testNodeClass,
        allPluginsPath=allPluginsPath,
        autoStart=True,
        do_post_node_creation=do_post_node_creation)
    yield looper, newNode, client, wallet, newStewardClient, \
        newStewardWallet
Example #8
0
def demote_primary_node(looper,
                        initial_pool_of_nodes,
                        pool_of_nodes,
                        poolTxnStewardNames,
                        poolTxnData,
                        tdirWithClientPoolTxns):
    demoted_node = [node for node in pool_of_nodes if node.has_master_primary][0]
    indx = initial_pool_of_nodes.index(demoted_node)
    steward_name = poolTxnStewardNames[indx]
    stewards_seed = poolTxnData["seeds"][steward_name].encode()

    stewardClient, stewardWallet = buildPoolClientAndWallet(
        (steward_name, stewards_seed), tdirWithClientPoolTxns)
    looper.add(stewardClient)
    looper.run(stewardClient.ensureConnectedToNodes())

    node_data = {
        ALIAS: demoted_node.name,
        SERVICES: []
    }
    updateNodeData(looper, stewardClient,
                   stewardWallet, demoted_node, node_data)
    pool_of_nodes = list(set(pool_of_nodes) - {demoted_node})

    return pool_of_nodes
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
                     tdirWithPoolTxns, poolTxnData, noRetryReq):
    """
    Client should not be using node registry but pool transaction file
    :return:
    """
    clients = []
    for name in poolTxnClientNames:
        seed = poolTxnData["seeds"][name].encode()
        client, wallet = buildPoolClientAndWallet((name, seed),
                                                  tdirWithPoolTxns)
        txnPoolNodesLooper.add(client)
        ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper,
                                                      client, *txnPoolNodeSet)
        clients.append((client, wallet))

    n = 250
    timeOutPerReq = 3
    for (client, wallet) in clients:
        logger.debug("{} sending {} requests".format(client, n))
        sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
                                            n, 1, timeOutPerReq)
        logger.debug("{} sent {} requests".format(client, n))
    for node in txnPoolNodeSet:
        logger.debug("{} has requests {} with size {}".format(
            node, len(node.requests), get_size(node.requests)))
        for replica in node.replicas:
            logger.debug("{} has prepares {} with size {}".format(
                replica, len(replica.prepares), get_size(replica.prepares)))
            logger.debug("{} has commits {} with size {}".format(
                replica, len(replica.commits), get_size(replica.commits)))
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
                     tdirWithPoolTxns, poolTxnData, noRetryReq):
    """
    Client should not be using node registry but pool transaction file
    :return:
    """
    clients = []
    for name in poolTxnClientNames:
        seed = poolTxnData["seeds"][name].encode()
        client, wallet = buildPoolClientAndWallet((name, seed),
                                                  tdirWithPoolTxns)
        txnPoolNodesLooper.add(client)
        ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper,
                                                      client, *txnPoolNodeSet)
        clients.append((client, wallet))

    numRequests = 250
    fVal = 1
    for (client, wallet) in clients:
        logger.debug("{} sending {} requests".format(client, numRequests))
        sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
                                  sdk_wallet_steward, numRequests)
        logger.debug("{} sent {} requests".format(client, numRequests))
    for node in txnPoolNodeSet:
        logger.debug("{} has requests {} with size {}".format(
            node, len(node.requests), get_size(node.requests)))
        for replica in node.replicas.values():
            logger.debug("{} has prepares {} with size {}".format(
                replica, len(replica.prepares), get_size(replica.prepares)))
            logger.debug("{} has commits {} with size {}".format(
                replica, len(replica.commits), get_size(replica.commits)))
Example #11
0
def clientAndWallet2(looper, poolTxnClientData, tdirWithClientPoolTxns):
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithClientPoolTxns)

    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    yield client, wallet
    client.stop()
Example #12
0
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet,
                                         tdirWithPoolTxns, tconf, steward1,
                                         stewardWallet, nodeThetaAdded,
                                         poolTxnClientData):
    newSteward, newStewardWallet, newNode = nodeThetaAdded
    nodeNewHa, clientNewHa = genHa(2)
    logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
                                                   clientNewHa))

    # Making the change HA txn an confirming its succeeded
    changeNodeHa(looper,
                 newSteward,
                 newStewardWallet,
                 newNode,
                 nodeHa=nodeNewHa,
                 clientHa=clientNewHa)

    # Stopping existing nodes
    for node in txnPoolNodeSet:
        node.stop()
        looper.removeProdable(node)

    # Starting nodes again by creating `Node` objects since that simulates
    # what happens when starting the node with script
    restartedNodes = []
    for node in txnPoolNodeSet[:-1]:
        restartedNode = TestNode(node.name,
                                 basedirpath=tdirWithPoolTxns,
                                 config=tconf,
                                 ha=node.nodestack.ha,
                                 cliha=node.clientstack.ha)
        looper.add(restartedNode)
        restartedNodes.append(restartedNode)

    # Starting the node whose HA was changed
    node = TestNode(newNode.name,
                    basedirpath=tdirWithPoolTxns,
                    config=tconf,
                    ha=nodeNewHa,
                    cliha=clientNewHa)
    looper.add(node)
    restartedNodes.append(node)

    looper.run(checkNodesConnected(restartedNodes))
    looper.run(
        eventually(checkNodeLedgersForEquality,
                   node,
                   *restartedNodes[:-1],
                   retryWait=1,
                   timeout=10))

    # Building a new client that reads from the genesis txn file
    # but is able to connect to all nodes
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *restartedNodes)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tdirWithPoolTxns,
                                         tdirWithClientPoolTxns, tconf, steward1,
                                         stewardWallet, nodeThetaAdded,
                                         poolTxnClientData):
    newSteward, newStewardWallet, newNode = nodeThetaAdded
    nodeNewHa, clientNewHa = genHa(2)
    logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
                                                   clientNewHa))

    # Making the change HA txn an confirming its succeeded
    op = {
        ALIAS: newNode.name,
        NODE_IP: nodeNewHa.host,
        NODE_PORT: nodeNewHa.port,
        CLIENT_IP: clientNewHa.host,
        CLIENT_PORT: clientNewHa.port,
    }
    updateNodeData(looper, newSteward, newStewardWallet, newNode,
                   op)

    # Stopping existing nodes
    for node in txnPoolNodeSet:
        node.stop()
        looper.removeProdable(node)

    # Starting nodes again by creating `Node` objects since that simulates
    # what happens when starting the node with script
    restartedNodes = []
    for node in txnPoolNodeSet[:-1]:
        config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir)
        restartedNode = TestNode(node.name,
                                 config_helper=config_helper,
                                 config=tconf, ha=node.nodestack.ha,
                                 cliha=node.clientstack.ha)
        looper.add(restartedNode)
        restartedNodes.append(restartedNode)

    # Starting the node whose HA was changed
    config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir)
    node = TestNode(newNode.name,
                    config_helper=config_helper,
                    config=tconf,
                    ha=nodeNewHa, cliha=clientNewHa)
    looper.add(node)
    restartedNodes.append(node)

    looper.run(checkNodesConnected(restartedNodes))
    waitNodeDataEquality(looper, node, *restartedNodes[:-1])

    # Building a new client that reads from the genesis txn file
    # but is able to connect to all nodes
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithClientPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *restartedNodes)
Example #14
0
def stewardClientAndWallet(poolNodesCreated, looper, tdirWithDomainTxns,
                           poolTxnStewardData):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithDomainTxns,
                                              clientClass=TestClient,
                                              walletClass=Wallet)
    client.registerObserver(wallet.handleIncomingReply)

    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    makePendingTxnsRequest(client, wallet)
    return client, wallet
Example #15
0
def poolAfterSomeTxns(looper, txnPoolNodesLooper, txnPoolNodeSet,
                      tdirWithClientPoolTxns, poolTxnStewardData,
                      allPluginsPath, request):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithClientPoolTxns,
                                              clientClass=TestClient)
    looper.run(checkNodesConnected(txnPoolNodeSet))
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    txnCount = getValueFromModule(request, "txnCount", 5)
    sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
                                        txnCount)
    yield looper, client, wallet
Example #16
0
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames,
                                  tdirWithClientPoolTxns, txnPoolNodeSet, txnPoolMasterNodes):
    primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet)
    master_node = txnPoolMasterNodes[0]
    stewardName = poolTxnStewardNames[primariesIdxs[0]]
    stewardsSeed = poolTxnData["seeds"][stewardName].encode()

    stewardClient, stewardWallet = buildPoolClientAndWallet(
        (stewardName, stewardsSeed), tdirWithClientPoolTxns)
    looper.add(stewardClient)
    looper.run(stewardClient.ensureConnectedToNodes())

    return stewardClient, stewardWallet
Example #17
0
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
                                   disable_node_monitor_config, looper,
                                   txnPoolNodeSet, tdirWithPoolTxns,
                                   allPluginsPath, poolTxnStewardData, capsys):

    # One of the reason memory grows is because spylog grows
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())

    client_batches = 300
    txns_per_batch = 25
    time_log = []
    warm_up_batches = 10
    tolerance_factor = 2
    from pympler import asizeof
    for i in range(client_batches):
        s = perf_counter()
        sendReqsToNodesAndVerifySuffReplies(looper,
                                            wallet,
                                            client,
                                            txns_per_batch,
                                            override_timeout_limit=True)
        t = perf_counter() - s
        with capsys.disabled():
            print('{} executed {} client txns in {:.2f} seconds'.format(
                i + 1, txns_per_batch, t))
            print('--------Memory Usage details start')
            for node in txnPoolNodeSet:
                # print(sys.getsizeof(node))
                print('---Node {}-----'.format(node))
                # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
                print(get_memory_usage(node, True, get_only_non_empty=True))
                for r in node.replicas:
                    print('---Replica {}-----'.format(r))
                    print(get_memory_usage(r, True, get_only_non_empty=True))
            print('--------Memory Usage details end')

        if len(time_log) >= warm_up_batches:
            m = mean(time_log)
            sd = tolerance_factor * pstdev(time_log)
            assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
        time_log.append(t)
        # Since client checks inbox for sufficient replies, clear inbox so that
        #  it takes constant time to check replies for each batch
        client.inBox.clear()
        client.txnLog.reset()
Example #18
0
def stewards_and_wallets(looper, txnPoolNodeSet, pool_txn_stewards_data,
                         tdirWithPoolTxns):
    clients_and_wallets = []
    for pool_txn_steward_data in pool_txn_stewards_data:
        steward_client, steward_wallet = buildPoolClientAndWallet(
            pool_txn_steward_data, tdirWithPoolTxns)
        looper.add(steward_client)
        ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client,
                                                      *txnPoolNodeSet)
        clients_and_wallets.append((steward_client, steward_wallet))

    yield clients_and_wallets

    for (client, wallet) in clients_and_wallets:
        client.stop()
Example #19
0
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf,
                                    tdirWithPoolTxns, allPluginsPath,
                                    poolTxnStewardData, capsys):

    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())

    nodes = txnPoolNodeSet
    x = nodes[-1]

    with capsys.disabled():
        print("Stopping node {} with pool ledger size {}".format(
            x, x.poolManager.txnSeqNo))

    disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x)
    looper.removeProdable(x)

    client_batches = 80
    txns_per_batch = 10
    for i in range(client_batches):
        s = perf_counter()
        sendReqsToNodesAndVerifySuffReplies(looper,
                                            wallet,
                                            client,
                                            txns_per_batch,
                                            override_timeout_limit=True)
        with capsys.disabled():
            print('{} executed {} client txns in {:.2f} seconds'.format(
                i + 1, txns_per_batch,
                perf_counter() - s))

    nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha)
    newNode = TestNode(x.name,
                       basedirpath=tdirWithPoolTxns,
                       base_data_dir=tdirWithPoolTxns,
                       config=tconf,
                       ha=nodeHa,
                       cliha=nodeCHa,
                       pluginPaths=allPluginsPath)
    looper.add(newNode)
    txnPoolNodeSet[-1] = newNode
    looper.run(checkNodesConnected(txnPoolNodeSet))
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet,
                                         tdirWithPoolTxns, tconf, steward1,
                                         stewardWallet, nodeThetaAdded,
                                         poolTxnClientData):
    newSteward, newStewardWallet, newNode = nodeThetaAdded
    nodeNewHa, clientNewHa = genHa(2)
    logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
                                                   clientNewHa))

    # Making the change HA txn an confirming its succeeded
    changeNodeHa(looper, newSteward, newStewardWallet, newNode,
                 nodeHa=nodeNewHa, clientHa=clientNewHa)

    # Stopping existing nodes
    for node in txnPoolNodeSet:
        node.stop()
        looper.removeProdable(node)

    # Starting nodes again by creating `Node` objects since that simulates
    # what happens when starting the node with script
    restartedNodes = []
    for node in txnPoolNodeSet[:-1]:
        restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns,
                                 config=tconf, ha=node.nodestack.ha,
                                 cliha=node.clientstack.ha)
        looper.add(restartedNode)
        restartedNodes.append(restartedNode)

    # Starting the node whose HA was changed
    node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf,
                    ha=nodeNewHa, cliha=clientNewHa)
    looper.add(node)
    restartedNodes.append(node)

    looper.run(checkNodesConnected(restartedNodes))
    looper.run(eventually(checkNodeLedgersForEquality, node,
                          *restartedNodes[:-1], retryWait=1, timeout=10))

    # Building a new client that reads from the genesis txn file
    # but is able to connect to all nodes
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    looper.add(client)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
                                                  *restartedNodes)
Example #21
0
def test_node_load(looper, txnPoolNodeSet, tconf,
                   tdirWithPoolTxns, allPluginsPath,
                   poolTxnStewardData, capsys):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())

    client_batches = 150
    txns_per_batch = 25
    for i in range(client_batches):
        s = perf_counter()
        sendReqsToNodesAndVerifySuffReplies(looper, wallet, client,
                                            txns_per_batch,
                                            override_timeout_limit=True)
        with capsys.disabled():
            print('{} executed {} client txns in {:.2f} seconds'.
                  format(i + 1, txns_per_batch, perf_counter() - s))
Example #22
0
def checkStewardAdded(looper, poolTxnStewardData, tdirWithPoolTxns):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    sigseed = b'55555555555555555555555555555555'
    newSigner = SimpleSigner(sigseed)
    op = {
        TXN_TYPE: NYM,
        ROLE: STEWARD,
        TARGET_NYM: newSigner.verkey,
        ALIAS: "Robert",
    }
    req = wallet.signOp(op)
    client.submitReqs(req)

    def chk():
        assert client.getReply(*req.key) == (None, "NOT_FOUND")

    looper.run(eventually(chk, retryWait=1, timeout=5))
Example #23
0
def checkStewardAdded(poolTxnStewardData, tdirWithPoolTxns):
    with Looper(debug=True) as looper:
        client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                                  tdirWithPoolTxns)
        looper.add(client)
        looper.run(client.ensureConnectedToNodes())
        sigseed = b'55555555555555555555555555555555'
        newSigner = SimpleSigner(sigseed)
        op = {
            TXN_TYPE: NYM,
            ROLE: STEWARD,
            TARGET_NYM: newSigner.verkey,
            ALIAS: "Robert",
        }
        req = wallet.signOp(op)
        client.submitReqs(req)

        def chk():
            assert client.getReply(req.reqId) == (None, "NOT_FOUND")

        looper.run(eventually(chk, retryWait=1, timeout=5))
Example #24
0
def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns,
              poolTxnStewardData):
    # TODO: Maybe this needs to be extracted in another fixture
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithClientPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())
    for i in range(tconf.ProcessedBatchMapsToKeep - 1):
        sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1)

    # All node maintain the same map from txn range to 3PC
    looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet))
    for i in range(3):
        sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1)

    # All node maintain the same map from txn range to 3PC and its equal to
    # `tconf.ProcessedBatchMapsToKeep` even after sending more batches than
    # `tconf.ProcessedBatchMapsToKeep`, which shows the garbage cleaning in
    # action
    looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet,
                          tconf.ProcessedBatchMapsToKeep))
Example #25
0
def nodeCreatedAfterSomeTxns(
    txnPoolNodesLooper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request
):
    # with Looper(debug=True) as looper:
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient)
    txnPoolNodesLooper.add(client)
    txnPoolNodesLooper.run(client.ensureConnectedToNodes())
    txnCount = getValueFromModule(request, "txnCount", 5)
    sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, txnCount, timeoutPerReq=25)

    newStewardName = randomString()
    newNodeName = "Epsilon"
    newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
        txnPoolNodesLooper,
        client,
        wallet,
        newStewardName,
        newNodeName,
        tdirWithPoolTxns,
        tconf,
        allPluginsPath=allPluginsPath,
        autoStart=True,
    )
    yield txnPoolNodesLooper, newNode, client, wallet, newStewardClient, newStewardWallet
Example #26
0
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
                                   disable_node_monitor_config, looper,
                                   txnPoolNodeSet, tdirWithPoolTxns,
                                   allPluginsPath, poolTxnStewardData, capsys):
    # One of the reason memory grows is because spylog grows
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns,
                                              clientClass=TestClient)
    looper.add(client)
    looper.run(client.ensureConnectedToNodes())

    client_batches = 300
    txns_per_batch = 25
    time_log = []
    warm_up_batches = 10
    tolerance_factor = 2
    print_detailed_memory_usage = False
    from pympler import tracker
    tr = tracker.SummaryTracker()
    node_methods_to_capture = [
        TestNode.executeBatch, TestNode.recordAndPropagate,
        TestNode.domainDynamicValidation, TestNode.domainRequestApplication
    ]
    times = {
        n.name: {meth.__name__: []
                 for meth in node_methods_to_capture}
        for n in txnPoolNodeSet
    }

    for node in txnPoolNodeSet:
        for meth in node_methods_to_capture:
            meth_name = meth.__name__
            patched = timeit(getattr(node, meth_name),
                             times[node.name][meth_name])
            setattr(node, meth_name, patched)

    for i in range(client_batches):
        s = perf_counter()
        sendReqsToNodesAndVerifySuffReplies(looper,
                                            wallet,
                                            client,
                                            txns_per_batch,
                                            override_timeout_limit=True)
        t = perf_counter() - s
        with capsys.disabled():
            print('{} executed {} client txns in {:.2f} seconds'.format(
                i + 1, txns_per_batch, t))
            print('--------Memory Usage details start')
            for node in txnPoolNodeSet:
                # print(sys.getsizeof(node))
                print('---Node {}-----'.format(node))
                # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
                print(
                    get_memory_usage(node,
                                     print_detailed_memory_usage,
                                     get_only_non_empty=True))
                for r in node.replicas:
                    print('---Replica {}-----'.format(r))
                    print(
                        get_memory_usage(r,
                                         print_detailed_memory_usage,
                                         get_only_non_empty=True))

            # if i % 3 == 0:
            #     tr.print_diff()
            print('--------Memory Usage details end')
            for node in txnPoolNodeSet:
                for meth in node_methods_to_capture:
                    ts = times[node.name][meth.__name__]
                    print('{} {} {} {}'.format(node, meth.__name__, mean(ts),
                                               ts))

        if len(time_log) >= warm_up_batches:
            m = mean(time_log)
            sd = tolerance_factor * pstdev(time_log)
            assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
        time_log.append(t)
        # Since client checks inbox for sufficient replies, clear inbox so that
        #  it takes constant time to check replies for each batch
        client.inBox.clear()
        client.txnLog.reset()
Example #27
0
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
    return buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns)
Example #28
0
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
                      tdirWithPoolTxns):
    return buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns)
def new_client(poolTxnClientData, tdirWithPoolTxns):
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    return (client, wallet)
Example #30
0
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
    client, wallet = buildPoolClientAndWallet(poolTxnClientData,
                                              tdirWithPoolTxns)
    yield client, wallet
    client.stop()
Example #31
0
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
                      tdirWithPoolTxns):
    client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
                                              tdirWithPoolTxns)
    yield client, wallet
    client.stop()
def testStewardSuspendsNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf,
                            steward1, stewardWallet, nodeThetaAdded,
                            poolTxnStewardData, allPluginsPath):

    newSteward, newStewardWallet, newNode = nodeThetaAdded
    newNodeNym = hexToFriendly(newNode.nodestack.local.signer.verhex)
    suspendNode(looper, newSteward, newStewardWallet, newNodeNym, newNode.name)
    # Check suspended node does not exist in any nodeReg or remotes of
    # nodes or clients

    txnPoolNodeSet = txnPoolNodeSet[:-1]
    for node in txnPoolNodeSet:
        looper.run(eventually(checkNodeNotInNodeReg, node, newNode.name))
    for client in (steward1, newSteward):
        looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name))

    # Check a client can send request and receive replies
    req = sendRandomRequest(newStewardWallet, newSteward)
    checkSufficientRepliesForRequests(looper,
                                      newSteward, [
                                          req,
                                      ],
                                      timeoutPerReq=10)

    # Check that a restarted client or node does not connect to the suspended
    # node
    steward1.stop()
    looper.removeProdable(steward1)
    steward1, stewardWallet = buildPoolClientAndWallet(poolTxnStewardData,
                                                       tdirWithPoolTxns)
    looper.add(steward1)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
                                                  *txnPoolNodeSet)
    looper.run(eventually(checkNodeNotInNodeReg, steward1, newNode.name))

    newNode.stop()
    looper.removeProdable(newNode)

    # TODO: There is a bug that if a primary node is turned off, it sends
    # duplicate Pre-Prepare and gets blacklisted. Here is the gist
    # https://gist.github.com/lovesh/c16989616ebb6856f9fa2905c14dc4b7
    oldNodeIdx, oldNode = [(i, n) for i, n in enumerate(txnPoolNodeSet)
                           if not n.hasPrimary][0]
    oldNode.stop()
    looper.removeProdable(oldNode)
    oldNode = TestNode(oldNode.name,
                       basedirpath=tdirWithPoolTxns,
                       config=tconf,
                       pluginPaths=allPluginsPath)
    looper.add(oldNode)
    txnPoolNodeSet[oldNodeIdx] = oldNode
    looper.run(checkNodesConnected(txnPoolNodeSet))
    looper.run(eventually(checkNodeNotInNodeReg, oldNode, newNode.name))

    # Check that a node whose suspension is revoked can reconnect to other
    # nodes and clients can also connect to that node
    cancelNodeSuspension(looper, newSteward, newStewardWallet, newNodeNym,
                         newNode.name)
    nodeTheta = TestNode(newNode.name,
                         basedirpath=tdirWithPoolTxns,
                         config=tconf,
                         pluginPaths=allPluginsPath,
                         ha=newNode.nodestack.ha,
                         cliha=newNode.clientstack.ha)
    looper.add(nodeTheta)
    txnPoolNodeSet.append(nodeTheta)
    looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=30))
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
                                                  *txnPoolNodeSet)
    ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
                                                  *txnPoolNodeSet)
Example #33
0
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
    return buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns)
Example #34
0
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
                      tdirWithPoolTxns):
    return buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns)