def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
config_helper = PNodeConfigHelper(name,
self.config,
chroot=self.tmpdir)
seed = randomSeed()
if self.keyshare:
learnKeysFromOthers(config_helper.keys_dir, name,
self.nodes.values())
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
config_helper=config_helper,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths,
seed=seed))
if self.keyshare:
tellKeysToOthers(node, self.nodes.values())
self.nodes[name] = node
self.__dict__[name] = node
return node
def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
config_helper = PNodeConfigHelper(name, self.config, chroot=self.tmpdir)
seed = randomSeed()
if self.keyshare:
learnKeysFromOthers(config_helper.keys_dir, name, self.nodes.values())
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
config_helper=config_helper,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths,
seed=seed))
if self.keyshare:
tellKeysToOthers(node, self.nodes.values())
self.nodes[name] = node
self.__dict__[name] = node
return node
def testNodeConnection(allPluginsPath, tdirAndLooper, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir, nrg)
logger.debug(names)
nodes = []
for name in names:
node = TestNode(name, nrg, basedirpath=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
nodes = []
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
for name in nodeReg:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func,
looper,
txnPoolNodeSetNotStarted):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
C = txnPoolNodeSetNotStarted[2]
for node in nodes:
tellKeysToOthers(node, [C, ])
looper.add(C)
looper.runFor(5)
stopNodes([C, ], looper)
stopNodes([A, B], looper)
for node in [A, B, C]:
looper.removeProdable(node)
def testNodeConnection(allPluginsPath, tconf, tdir,
tdir_for_func, tconf_for_func,
looper, txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
looper.removeProdable(A)
looper.removeProdable(B)
A = start_stopped_node(A, looper, tconf, tdir, allPluginsPath)
looper.runFor(4)
B = start_stopped_node(B, looper, tconf, tdir, allPluginsPath)
looper.run(checkNodesConnected([A, B]))
for node in txnPoolNodeSetNotStarted[2:]:
looper.add(node)
all_nodes = [A, B] + txnPoolNodeSetNotStarted[2:]
looper.run(checkNodesConnected(all_nodes))
stopNodes(all_nodes, looper)
for node in all_nodes:
looper.removeProdable(node)
def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
seed = randomSeed()
if self.keyshare:
learnKeysFromOthers(self.tmpdir, name, self.nodes.values())
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
nodeRegistry=copy(self.nodeReg),
basedirpath=self.tmpdir,
base_data_dir=self.tmpdir,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths,
seed=seed))
if self.keyshare:
tellKeysToOthers(node, self.nodes.values())
self.nodes[name] = node
self.__dict__[name] = node
return node
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func, looper,
txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
for node in [A, B]:
looper.removeProdable(node)
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg, conf):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir, nrg)
logger.debug(names)
nodes = []
for name in names:
node = TestNode(name,
nrg,
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
initLocalKeys(tdir, {"Gamma": nodeReg["Gamma"]})
C = TestNode("Gamma", {
**nrg,
**{
"Gamma": nodeReg["Gamma"]
}
},
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
for node in nodes:
tellKeysToOthers(node, [
C,
])
looper.add(C)
looper.runFor(5)
stopNodes([
C,
], looper)
def chk():
assert C.name not in B.nodestack.nameRemotes
assert C.name not in A.nodestack.nameRemotes
timeout = waits.expectedPoolInterconnectionTime(len(nodeReg))
looper.run(eventually(chk, retryWait=2, timeout=timeout))
stopNodes([A, B], looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
# # Giving some time for sockets to close, use eventually
# time.sleep(1)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
def testNodesComingUpAtDifferentTimes(allPluginsPath, tconf, tdir,
tdir_for_func, tconf_for_func,
looper, txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted
names = list(node.name for node in nodes)
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
current_node_set = set(nodes)
for node in nodes:
disconnect_node_and_ensure_disconnected(looper,
current_node_set,
node,
timeout=len(nodes),
stopNode=True)
looper.removeProdable(node)
current_node_set.remove(node)
for i, node in enumerate(nodes):
restarted_node = start_stopped_node(node, looper,
tconf, tdir, allPluginsPath)
current_node_set.add(restarted_node)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(current_node_set))
stopNodes(current_node_set, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
for node in current_node_set:
looper.removeProdable(node)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func, tconf_for_func,
looper,
txnPoolNodeSetNotStarted):
nodes = txnPoolNodeSetNotStarted
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
for node in nodes:
looper.removeProdable(node)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], numInstances=2)
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func)
node = TestNode(name, nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], instances_list=range(2))
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
for node in nodes:
looper.removeProdable(node)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdirAndLooper,
nodeReg):
tdir, looper = tdirAndLooper
nodes = []
initLocalKeys(tdir, nodeReg)
for name in nodeReg:
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func,
tconf_for_func, looper,
txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted
names = list(node.name for node in nodes)
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
# # Giving some time for sockets to close, use eventually
# time.sleep(1)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
for node in nodes:
looper.removeProdable(node)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdirAndLooper, nodeReg):
tdir, looper = tdirAndLooper
initLocalKeys(tdir, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = TestNode(name,
nodeReg,
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], numInstances=2)
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
looper = looper_without_nodeset_for_func
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir_for_func, tconf_for_func, nrg)
logger.debug(names)
nodes = []
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nrg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func, nodeReg):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
looper = looper_without_nodeset_for_func
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir_for_func, tconf_for_func, nrg)
logger.debug(names)
nodes = []
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nrg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
name = "Gamma"
initLocalKeys(tdir_for_func, tconf_for_func, {name: nodeReg[name]})
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
C = TestNode(name, {
**nrg,
**{
name: nodeReg[name]
}
},
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
for node in nodes:
tellKeysToOthers(node, [
C,
])
looper.add(C)
looper.runFor(5)
stopNodes([
C,
], looper)
timeout = waits.expectedPoolInterconnectionTime(len(nodeReg))
stopNodes([A, B], looper)