def addNode(self, name: str) -> TestNode: if name in self.nodes: error("{} already added".format(name)) assert name in self.nodeReg ha, cliname, cliha = self.nodeReg[name] config_helper = PNodeConfigHelper(name, self.config, chroot=self.tmpdir) seed = randomSeed() if self.keyshare: learnKeysFromOthers(config_helper.keys_dir, name, self.nodes.values()) testNodeClass = self.testNodeClass node = self.enter_context( testNodeClass(name=name, ha=ha, cliname=cliname, cliha=cliha, config_helper=config_helper, primaryDecider=self.primaryDecider, pluginPaths=self.pluginPaths, seed=seed)) if self.keyshare: tellKeysToOthers(node, self.nodes.values()) self.nodes[name] = node self.__dict__[name] = node return node
def addNode(self, name: str) -> TestNode: if name in self.nodes: error("{} already added".format(name)) assert name in self.nodeReg ha, cliname, cliha = self.nodeReg[name] config_helper = PNodeConfigHelper(name, self.config, chroot=self.tmpdir) seed = randomSeed() if self.keyshare: learnKeysFromOthers(config_helper.keys_dir, name, self.nodes.values()) testNodeClass = self.testNodeClass node = self.enter_context( testNodeClass(name=name, ha=ha, cliname=cliname, cliha=cliha, config_helper=config_helper, primaryDecider=self.primaryDecider, pluginPaths=self.pluginPaths, seed=seed)) if self.keyshare: tellKeysToOthers(node, self.nodes.values()) self.nodes[name] = node self.__dict__[name] = node return node
def testNodeConnection(allPluginsPath, tdirAndLooper, nodeReg): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) tdir, looper = tdirAndLooper names = ["Alpha", "Beta"] nrg = {n: nodeReg[n] for n in names} initLocalKeys(tdir, nrg) logger.debug(names) nodes = [] for name in names: node = TestNode(name, nrg, basedirpath=tdir, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes looper.add(A) looper.runFor(4) logger.debug("wait done") looper.add(B) looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() A.start(looper.loop) looper.runFor(4) B.start(looper.loop) looper.run(checkNodesConnected([A, B])) stopNodes([A, B], looper)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): looper = looper_without_nodeset_for_func nodes = [] initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) for name in nodeReg: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): """ The nodes Alpha and Beta know about each other so they should connect but they should remove remote for C when it tries to connect to them """ nodes = txnPoolNodeSetNotStarted[:2] for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) C = txnPoolNodeSetNotStarted[2] for node in nodes: tellKeysToOthers(node, [C, ]) looper.add(C) looper.runFor(5) stopNodes([C, ], looper) stopNodes([A, B], looper) for node in [A, B, C]: looper.removeProdable(node)
def testNodeConnection(allPluginsPath, tconf, tdir, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted[:2] for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes looper.add(A) looper.runFor(4) logger.debug("wait done") looper.add(B) looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() looper.removeProdable(A) looper.removeProdable(B) A = start_stopped_node(A, looper, tconf, tdir, allPluginsPath) looper.runFor(4) B = start_stopped_node(B, looper, tconf, tdir, allPluginsPath) looper.run(checkNodesConnected([A, B])) for node in txnPoolNodeSetNotStarted[2:]: looper.add(node) all_nodes = [A, B] + txnPoolNodeSetNotStarted[2:] looper.run(checkNodesConnected(all_nodes)) stopNodes(all_nodes, looper) for node in all_nodes: looper.removeProdable(node)
def addNode(self, name: str) -> TestNode: if name in self.nodes: error("{} already added".format(name)) assert name in self.nodeReg ha, cliname, cliha = self.nodeReg[name] seed = randomSeed() if self.keyshare: learnKeysFromOthers(self.tmpdir, name, self.nodes.values()) testNodeClass = self.testNodeClass node = self.enter_context( testNodeClass(name=name, ha=ha, cliname=cliname, cliha=cliha, nodeRegistry=copy(self.nodeReg), basedirpath=self.tmpdir, base_data_dir=self.tmpdir, primaryDecider=self.primaryDecider, pluginPaths=self.pluginPaths, seed=seed)) if self.keyshare: tellKeysToOthers(node, self.nodes.values()) self.nodes[name] = node self.__dict__[name] = node return node
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted[:2] for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes looper.add(A) looper.runFor(4) logger.debug("wait done") looper.add(B) looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() A.start(looper.loop) looper.runFor(4) B.start(looper.loop) looper.run(checkNodesConnected([A, B])) stopNodes([A, B], looper) for node in [A, B]: looper.removeProdable(node)
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg, conf): """ The nodes Alpha and Beta know about each other so they should connect but they should remove remote for C when it tries to connect to them """ tdir, looper = tdirAndLooper names = ["Alpha", "Beta"] nrg = {n: nodeReg[n] for n in names} initLocalKeys(tdir, nrg) logger.debug(names) nodes = [] for name in names: node = TestNode(name, nrg, basedirpath=tdir, base_data_dir=tdir, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) initLocalKeys(tdir, {"Gamma": nodeReg["Gamma"]}) C = TestNode("Gamma", { **nrg, **{ "Gamma": nodeReg["Gamma"] } }, basedirpath=tdir, base_data_dir=tdir, pluginPaths=allPluginsPath) for node in nodes: tellKeysToOthers(node, [ C, ]) looper.add(C) looper.runFor(5) stopNodes([ C, ], looper) def chk(): assert C.name not in B.nodestack.nameRemotes assert C.name not in A.nodestack.nameRemotes timeout = waits.expectedPoolInterconnectionTime(len(nodeReg)) looper.run(eventually(chk, retryWait=2, timeout=timeout)) stopNodes([A, B], looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) looper = looper_without_nodeset_for_func initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) nodes = [] names = list(nodeReg.keys()) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for name in names: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) for i, node in enumerate(nodes): looper.add(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes)) logger.debug("connects") logger.debug("node order: {}".format(names)) logger.debug("waits: {}".format(waits)) stopNodes(nodes, looper) # # Giving some time for sockets to close, use eventually # time.sleep(1) for i, n in enumerate(nodes): n.start(looper.loop) looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper) logger.debug("reconnects") logger.debug("node order: {}".format(names)) logger.debug("rwaits: {}".format(rwaits))
def testNodesComingUpAtDifferentTimes(allPluginsPath, tconf, tdir, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted names = list(node.name for node in nodes) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for node in nodes: tellKeysToOthers(node, nodes) for i, node in enumerate(nodes): looper.add(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes)) logger.debug("connects") logger.debug("node order: {}".format(names)) logger.debug("waits: {}".format(waits)) current_node_set = set(nodes) for node in nodes: disconnect_node_and_ensure_disconnected(looper, current_node_set, node, timeout=len(nodes), stopNode=True) looper.removeProdable(node) current_node_set.remove(node) for i, node in enumerate(nodes): restarted_node = start_stopped_node(node, looper, tconf, tdir, allPluginsPath) current_node_set.add(restarted_node) looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(current_node_set)) stopNodes(current_node_set, looper) logger.debug("reconnects") logger.debug("node order: {}".format(names)) logger.debug("rwaits: {}".format(rwaits)) for node in current_node_set: looper.removeProdable(node)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): nodes = txnPoolNodeSetNotStarted for node in nodes: tellKeysToOthers(node, nodes) for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper) for node in nodes: looper.removeProdable(node)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): looper = looper_without_nodeset_for_func initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) nodes = [] names = list(nodeReg.keys()) logger.debug("Node names: {}".format(names)) def create(name): config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) return node for name in names: create(name) logger.debug("Creating keys") for node in nodes: tellKeysToOthers(node, nodes) for node in nodes[:3]: looper.add(node) looper.run(checkNodesConnected(nodes[:3])) # wait for the election to complete with the first three nodes ensureElectionsDone(looper, nodes[:3], numInstances=2) # start the fourth and see that it learns who the primaries are # from the other nodes looper.add(nodes[3]) # ensure election is done for updated pool ensureElectionsDone(looper, nodes) stopNodes(nodes, looper)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): looper = looper_without_nodeset_for_func initLocalKeys(tdir_for_func, tconf_for_func, nodeReg) nodes = [] names = list(nodeReg.keys()) logger.debug("Node names: {}".format(names)) def create(name): config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nodeReg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) return node for name in names: create(name) logger.debug("Creating keys") for node in nodes: tellKeysToOthers(node, nodes) for node in nodes[:3]: looper.add(node) looper.run(checkNodesConnected(nodes[:3])) # wait for the election to complete with the first three nodes ensureElectionsDone(looper, nodes[:3], instances_list=range(2)) # start the fourth and see that it learns who the primaries are # from the other nodes looper.add(nodes[3]) # ensure election is done for updated pool ensureElectionsDone(looper, nodes) stopNodes(nodes, looper) for node in nodes: looper.removeProdable(node)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdirAndLooper, nodeReg): tdir, looper = tdirAndLooper nodes = [] initLocalKeys(tdir, nodeReg) for name in nodeReg: node = TestNode(name, nodeReg, basedirpath=tdir, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted names = list(node.name for node in nodes) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for node in nodes: tellKeysToOthers(node, nodes) for i, node in enumerate(nodes): looper.add(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes)) logger.debug("connects") logger.debug("node order: {}".format(names)) logger.debug("waits: {}".format(waits)) stopNodes(nodes, looper) # # Giving some time for sockets to close, use eventually # time.sleep(1) for i, n in enumerate(nodes): n.start(looper.loop) looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(nodes)) stopNodes(nodes, looper) logger.debug("reconnects") logger.debug("node order: {}".format(names)) logger.debug("rwaits: {}".format(rwaits)) for node in nodes: looper.removeProdable(node)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdirAndLooper, nodeReg): tdir, looper = tdirAndLooper initLocalKeys(tdir, nodeReg) nodes = [] names = list(nodeReg.keys()) logger.debug("Node names: {}".format(names)) def create(name): node = TestNode(name, nodeReg, basedirpath=tdir, base_data_dir=tdir, pluginPaths=allPluginsPath) nodes.append(node) return node for name in names: create(name) logger.debug("Creating keys") for node in nodes: tellKeysToOthers(node, nodes) for node in nodes[:3]: looper.add(node) looper.run(checkNodesConnected(nodes[:3])) # wait for the election to complete with the first three nodes ensureElectionsDone(looper, nodes[:3], numInstances=2) # start the fourth and see that it learns who the primaries are # from the other nodes looper.add(nodes[3]) # ensure election is done for updated pool ensureElectionsDone(looper, nodes) stopNodes(nodes, looper)
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) looper = looper_without_nodeset_for_func names = ["Alpha", "Beta"] nrg = {n: nodeReg[n] for n in names} initLocalKeys(tdir_for_func, tconf_for_func, nrg) logger.debug(names) nodes = [] for name in names: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nrg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes looper.add(A) looper.runFor(4) logger.debug("wait done") looper.add(B) looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() A.start(looper.loop) looper.runFor(4) B.start(looper.loop) looper.run(checkNodesConnected([A, B])) stopNodes([A, B], looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func, looper_without_nodeset_for_func, nodeReg): """ The nodes Alpha and Beta know about each other so they should connect but they should remove remote for C when it tries to connect to them """ looper = looper_without_nodeset_for_func names = ["Alpha", "Beta"] nrg = {n: nodeReg[n] for n in names} initLocalKeys(tdir_for_func, tconf_for_func, nrg) logger.debug(names) nodes = [] for name in names: config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) node = TestNode(name, nrg, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) nodes.append(node) for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) name = "Gamma" initLocalKeys(tdir_for_func, tconf_for_func, {name: nodeReg[name]}) config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func) C = TestNode(name, { **nrg, **{ name: nodeReg[name] } }, config_helper=config_helper, config=tconf_for_func, pluginPaths=allPluginsPath) for node in nodes: tellKeysToOthers(node, [ C, ]) looper.add(C) looper.runFor(5) stopNodes([ C, ], looper) timeout = waits.expectedPoolInterconnectionTime(len(nodeReg)) stopNodes([A, B], looper)