def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdirAndLooper, nodeReg):
tdir, looper = tdirAndLooper
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
node.startKeySharing()
nodes.append(node)
for name in names[:3]:
create(name)
looper.run(checkNodesConnected(nodes))
# wait for the election to complete with the first three nodes
looper.runFor(10)
# create the fourth and see that it learns who the primaries are
# from the other nodes
create(names[3])
checkProtocolInstanceSetup(looper, nodes, timeout=10)
stopNodes(nodes, looper)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
nodes = []
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
for name in nodeReg:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def stop_nodes_and_remove_first(looper, nodes):
first_node = nodes[0]
stopNodes(nodes, looper)
looper.removeProdable(first_node)
looper.runFor(3) # let the nodes stop
return first_node, \
list(filter(lambda x: x.name != first_node.name, nodes))
def testViewNotChangedIfBackupPrimaryDisconnected(txnPoolNodeSet,
txnPoolNodesLooper, tconf):
"""
View change does not occurs when backup's primary is disconnected
"""
# Setup
nodes = txnPoolNodeSet
looper = txnPoolNodesLooper
viewNoBefore = checkViewNoForNodes(nodes)
primaryNodeForBackupInstance1Before = nodeByName(
nodes, primaryNodeNameForInstance(nodes, 1))
# Exercise
stopNodes([primaryNodeForBackupInstance1Before], looper)
# Verify
remainingNodes = set(nodes) - {primaryNodeForBackupInstance1Before}
looper.runFor(tconf.ToleratePrimaryDisconnection + 2)
def assertNewPrimariesElected():
with pytest.raises(AssertionError):
assert checkViewNoForNodes(remainingNodes) == viewNoBefore + 1
viewNoAfter = checkViewNoForNodes(remainingNodes, viewNoBefore)
assert viewNoBefore == viewNoAfter
looper.run(eventually(assertNewPrimariesElected, retryWait=1, timeout=30))
def stop_nodes_and_remove_first(looper, nodes):
first_node = nodes[0]
stopNodes(nodes, looper)
looper.removeProdable(first_node)
looper.runFor(3) # let the nodes stop
return first_node, \
list(filter(lambda x: x.name != first_node.name, nodes))
def testViewChangesIfMasterPrimaryDisconnected(txnPoolNodeSet, looper, wallet1,
client1, client1Connected,
tconf):
"""
View change occurs when master's primary is disconnected
"""
# Setup
nodes = txnPoolNodeSet
viewNoBefore = checkViewNoForNodes(nodes)
old_pr_node = get_master_primary_node(nodes)
# Stop primary
stopNodes([old_pr_node], looper)
looper.removeProdable(old_pr_node)
remainingNodes = list(set(nodes) - {old_pr_node})
# Sometimes it takes time for nodes to detect disconnection
ensure_node_disconnected(looper, old_pr_node, remainingNodes, timeout=20)
looper.runFor(tconf.ToleratePrimaryDisconnection + 2)
# Give some time to detect disconnection and then verify that view has
# changed and new primary has been elected
waitForViewChange(looper, remainingNodes, viewNoBefore + 1)
ensure_all_nodes_have_same_data(looper, nodes=remainingNodes)
new_pr_node = get_master_primary_node(remainingNodes)
assert old_pr_node != new_pr_node
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
def testNodeConnection(allPluginsPath, tdirAndLooper, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir, nrg)
logger.debug(names)
nodes = []
for name in names:
node = TestNode(name,
nrg,
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func,
looper,
txnPoolNodeSetNotStarted):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
C = txnPoolNodeSetNotStarted[2]
for node in nodes:
tellKeysToOthers(node, [C, ])
looper.add(C)
looper.runFor(5)
stopNodes([C, ], looper)
stopNodes([A, B], looper)
for node in [A, B, C]:
looper.removeProdable(node)
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
logger.debug(names)
nrg = {n: nodeReg[n] for n in names}
A, B = [TestNode(name, nrg, basedirpath=tdir,
pluginPaths=allPluginsPath)
for name in names]
for node in (A, B):
looper.add(node)
node.startKeySharing()
looper.run(checkNodesConnected([A, B]))
C = TestNode("Gamma", {**nrg, **{"Gamma": nodeReg["Gamma"]}},
basedirpath=tdir, pluginPaths=allPluginsPath)
looper.add(C)
C.startKeySharing(timeout=20)
def chk():
assert not C.nodestack.isKeySharing
looper.run(eventually(chk, retryWait=2, timeout=21))
stopNodes([C, ], looper)
def chk():
assert C.name not in B.nodestack.nameRemotes
assert C.name not in A.nodestack.nameRemotes
looper.run(eventually(chk, retryWait=2, timeout=5))
stopNodes([A, B], looper)
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func, looper,
txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
for node in [A, B]:
looper.removeProdable(node)
def testNodeConnection(allPluginsPath, tconf, tdir,
tdir_for_func, tconf_for_func,
looper, txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted[:2]
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
looper.removeProdable(A)
looper.removeProdable(B)
A = start_stopped_node(A, looper, tconf, tdir, allPluginsPath)
looper.runFor(4)
B = start_stopped_node(B, looper, tconf, tdir, allPluginsPath)
looper.run(checkNodesConnected([A, B]))
for node in txnPoolNodeSetNotStarted[2:]:
looper.add(node)
all_nodes = [A, B] + txnPoolNodeSetNotStarted[2:]
looper.run(checkNodesConnected(all_nodes))
stopNodes(all_nodes, looper)
for node in all_nodes:
looper.removeProdable(node)
def testNodeRemoveUnknownRemote(allPluginsPath, tdirAndLooper, nodeReg, conf):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir, nrg)
logger.debug(names)
nodes = []
for name in names:
node = TestNode(name,
nrg,
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
initLocalKeys(tdir, {"Gamma": nodeReg["Gamma"]})
C = TestNode("Gamma", {
**nrg,
**{
"Gamma": nodeReg["Gamma"]
}
},
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
for node in nodes:
tellKeysToOthers(node, [
C,
])
looper.add(C)
looper.runFor(5)
stopNodes([
C,
], looper)
def chk():
assert C.name not in B.nodestack.nameRemotes
assert C.name not in A.nodestack.nameRemotes
timeout = waits.expectedPoolInterconnectionTime(len(nodeReg))
looper.run(eventually(chk, retryWait=2, timeout=timeout))
stopNodes([A, B], looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
# # Giving some time for sockets to close, use eventually
# time.sleep(1)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdirAndLooper, nodeReg):
tdir, looper = tdirAndLooper
nodes = []
for name in nodeReg:
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
node.startKeySharing()
nodes.append(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tconf, tdir,
tdir_for_func, tconf_for_func,
looper, txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted
names = list(node.name for node in nodes)
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
current_node_set = set(nodes)
for node in nodes:
disconnect_node_and_ensure_disconnected(looper,
current_node_set,
node,
timeout=len(nodes),
stopNode=True)
looper.removeProdable(node)
current_node_set.remove(node)
for i, node in enumerate(nodes):
restarted_node = start_stopped_node(node, looper,
tconf, tdir, allPluginsPath)
current_node_set.add(restarted_node)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(current_node_set))
stopNodes(current_node_set, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
for node in current_node_set:
looper.removeProdable(node)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdir_for_func, tconf_for_func,
looper,
txnPoolNodeSetNotStarted):
nodes = txnPoolNodeSetNotStarted
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
for node in nodes:
looper.removeProdable(node)
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
tdir):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
npr = [n for n in txnPoolNodeSet if not n.hasPrimary]
nodeToCrash = npr[0]
idxToCrash = txnPoolNodeSet.index(nodeToCrash)
otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash]
def checkFlakyConnected(conn=True):
for node in otherNodes:
if conn:
assert nodeToCrash.nodestack.name in node.nodestack.connecteds
else:
assert nodeToCrash.nodestack.name not in node.nodestack.connecteds
checkFlakyConnected(True)
nodeToCrash.stop()
logger.debug('Stopped node {}'.format(nodeToCrash))
looper.removeProdable(nodeToCrash)
looper.runFor(1)
stopNodes([nodeToCrash], looper)
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=60))
looper.runFor(1)
config_helper = PNodeConfigHelper(nodeToCrash.name, tconf, chroot=tdir)
node = TestNode(
nodeToCrash.name,
config_helper=config_helper,
config=tconf,
ha=nodeToCrash.nodestack.ha,
cliha=nodeToCrash.clientstack.ha)
looper.add(node)
txnPoolNodeSet[idxToCrash] = node
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50))
ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 10)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func,
tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], numInstances=2)
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdirAndLooper,
nodeReg, conf):
tdir, looper = tdirAndLooper
nodes = []
if conf.UseZStack:
names = list(nodeReg.keys())
genKeys(tdir, names + [_ + CLIENT_STACK_SUFFIX for _ in names])
for name in nodeReg:
node = TestNode(name,
nodeReg,
basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
nodes.append(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func,
nodeReg):
looper = looper_without_nodeset_for_func
initLocalKeys(tdir_for_func, tconf_for_func, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
config_helper = PNodeConfigHelper(name, tconf_for_func, chroot=tdir_for_func)
node = TestNode(name, nodeReg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], instances_list=range(2))
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
for node in nodes:
looper.removeProdable(node)
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1,
tdir, client1Connected):
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
npr = [n for n in txnPoolNodeSet if not n.hasPrimary]
nodeToCrash = npr[0]
idxToCrash = txnPoolNodeSet.index(nodeToCrash)
otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash]
def checkFlakyConnected(conn=True):
for node in otherNodes:
if conn:
assert nodeToCrash.nodestack.name in node.nodestack.connecteds
else:
assert nodeToCrash.nodestack.name not in node.nodestack.connecteds
checkFlakyConnected(True)
nodeToCrash.stop()
logger.debug('Stopped node {}'.format(nodeToCrash))
looper.removeProdable(nodeToCrash)
looper.runFor(1)
stopNodes([nodeToCrash], looper)
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=60))
looper.runFor(1)
config_helper = PNodeConfigHelper(nodeToCrash.name, tconf, chroot=tdir)
node = TestNode(nodeToCrash.name,
ledger_dir=config_helper.ledger_dir,
keys_dir=config_helper.keys_dir,
genesis_dir=config_helper.genesis_dir,
plugins_dir=config_helper.plugins_dir,
config=tconf,
ha=nodeToCrash.nodestack.ha,
cliha=nodeToCrash.clientstack.ha)
looper.add(node)
txnPoolNodeSet[idxToCrash] = node
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50))
ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdirAndLooper, nodeReg,
conf):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
nodes = []
names = list(nodeReg.keys())
if conf.UseZStack:
genKeys(tdir, names + [_ + CLIENT_STACK_SUFFIX for _ in names])
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for i, name in enumerate(names):
node = TestNode(name,
nodeReg,
basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
nodes.append(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes, overrideTimeout=10))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
# # Giving some time for sockets to close, use eventually
# time.sleep(1)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes, overrideTimeout=10))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
def testNodesConnectWhenTheyAllStartAtOnce(allPluginsPath, tdirAndLooper,
nodeReg):
tdir, looper = tdirAndLooper
nodes = []
initLocalKeys(tdir, nodeReg)
for name in nodeReg:
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdir_for_func,
tconf_for_func, looper,
txnPoolNodeSetNotStarted):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
nodes = txnPoolNodeSetNotStarted
names = list(node.name for node in nodes)
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for node in nodes:
tellKeysToOthers(node, nodes)
for i, node in enumerate(nodes):
looper.add(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
# # Giving some time for sockets to close, use eventually
# time.sleep(1)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
for node in nodes:
looper.removeProdable(node)
def test_client_sends_get_request_to_one_node(looper,
client1,
wallet1,
nodeSet):
"""
Check that read only equest can be sent
without having connection to all nodes
"""
client = client1
wallet = wallet1
logger.info("Stopping nodes")
nodes_to_stop = list(nodeSet)[1:]
stopNodes(nodes_to_stop, looper)
def sign_and_send(op):
signed = wallet.signOp(op)
send_signed_requests(client, [signed])
def check_client_disconnected():
assert not client.hasSufficientConnections
logger.info("Ensure that client has disconnected from the stopped nodes")
# TODO non-default timeout
looper.run(eventually(check_client_disconnected, retryWait=1))
initial_submit_count = client.spylog.count(client.submitReqs)
initial_send_count = client.spylog.count(client.send)
logger.info("Send set request")
buy = {'type': 'buy', 'amount': random.randint(10, 100)}
sign_and_send(buy)
assert initial_submit_count + 1 == client.spylog.count(client.submitReqs)
assert initial_send_count == client.spylog.count(client.send)
logger.info("Send get request")
get_buy = {'type': 'get_buy'}
client._read_only_requests.add('get_buy')
sign_and_send(get_buy)
assert initial_submit_count + 2 == client.spylog.count(client.submitReqs)
assert initial_send_count + 1 == client.spylog.count(client.send)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdirAndLooper, nodeReg):
tdir, looper = tdirAndLooper
initLocalKeys(tdir, nodeReg)
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = TestNode(name,
nodeReg,
basedirpath=tdir,
base_data_dir=tdir,
pluginPaths=allPluginsPath)
nodes.append(node)
return node
for name in names:
create(name)
logger.debug("Creating keys")
for node in nodes:
tellKeysToOthers(node, nodes)
for node in nodes[:3]:
looper.add(node)
looper.run(checkNodesConnected(nodes[:3]))
# wait for the election to complete with the first three nodes
ensureElectionsDone(looper, nodes[:3], numInstances=2)
# start the fourth and see that it learns who the primaries are
# from the other nodes
looper.add(nodes[3])
# ensure election is done for updated pool
ensureElectionsDone(looper, nodes)
stopNodes(nodes, looper)
def testNodeConnection(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
looper = looper_without_nodeset_for_func
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir_for_func, tconf_for_func, nrg)
logger.debug(names)
nodes = []
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nrg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
looper.add(A)
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def testNodeConnectionAfterKeysharingRestarted(allPluginsPath, tdirAndLooper):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
timeout = 60
names = ["Alpha", "Beta"]
logger.debug(names)
nrg = {n: nodeReg[n] for n in names}
A, B = [TestNode(name, nodeRegistry=nrg, basedirpath=tdir,
pluginPaths=allPluginsPath)
for name in names]
looper.add(A)
A.startKeySharing(timeout=timeout)
looper.runFor(timeout+1)
logger.debug("done waiting for A's timeout")
looper.add(B)
B.startKeySharing(timeout=timeout)
looper.runFor(timeout+1)
logger.debug("done waiting for B's timeout")
A.startKeySharing(timeout=timeout)
B.startKeySharing(timeout=timeout)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def test_view_change_n_minus_f_quorum(txnPoolNodeSet, looper):
"""
Check that quorum n - f is used for view change
"""
assert len(txnPoolNodeSet) == 4
# Quorum for view change is expected to be n - f
# So, switching one node off
stopped = [txnPoolNodeSet[-1]]
active = list(txnPoolNodeSet)[:-1]
stopNodes(stopped, looper)
looper.removeProdable(*stopped)
# Check that view changes
ensure_view_change(looper, active)
ensureElectionsDone(looper=looper, nodes=active,
instances_list=range(2), customTimeout=60)
ensure_all_nodes_have_same_data(looper, nodes=active)
# Switching another node off to make sure that this time the quorum is not
# reaches.
stopped = [active[-1]]
active = list(active)[:-1]
stopNodes(stopped, looper)
looper.removeProdable(*stopped)
# Check that view does not changes
current_view_no = active[0].viewNo
with pytest.raises(AssertionError,
message="{} == {}, "
"Alpha -> Ratio: None,"
"Beta -> Ratio: None,"
"Delta -> Ratio: None"
.format(current_view_no,
current_view_no + 1)) as exc_info:
ensure_view_change(looper, active)
def test_view_change_n_minus_f_quorum(txnPoolNodeSet, looper):
"""
Check that quorum n - f is used for view change
"""
assert len(txnPoolNodeSet) == 4
# Quorum for view change is expected to be n - f
# So, switching one node off
stopped = [txnPoolNodeSet[-1]]
active = list(txnPoolNodeSet)[:-1]
stopNodes(stopped, looper)
looper.removeProdable(*stopped)
# Check that view changes
ensure_view_change(looper, active)
ensureElectionsDone(looper=looper, nodes=active,
numInstances=2, customTimeout=60)
ensure_all_nodes_have_same_data(looper, nodes=active)
# Switching another node off to make sure that this time the quorum is not
# reaches.
stopped = [active[-1]]
active = list(active)[:-1]
stopNodes(stopped, looper)
looper.removeProdable(*stopped)
# Check that view does not changes
current_view_no = active[0].viewNo
with pytest.raises(AssertionError,
message="{} == {}, "
"Alpha -> Ratio: None,"
"Beta -> Ratio: None,"
"Delta -> Ratio: None"
.format(current_view_no,
current_view_no + 1)) as exc_info:
ensure_view_change(looper, active)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tdirAndLooper, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
nodes = []
names = list(nodeReg.keys())
shuffle(names)
waits = [randint(1, 10) for _ in names]
rwaits = [randint(1, 10) for _ in names]
for i, name in enumerate(names):
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
node.startKeySharing()
nodes.append(node)
looper.runFor(waits[i])
looper.run(checkNodesConnected(nodes,
overrideTimeout=10))
logger.debug("connects")
logger.debug("node order: {}".format(names))
logger.debug("waits: {}".format(waits))
stopNodes(nodes, looper)
for i, n in enumerate(nodes):
n.start(looper.loop)
looper.runFor(rwaits[i])
looper.runFor(3)
looper.run(checkNodesConnected(nodes,
overrideTimeout=10))
stopNodes(nodes, looper)
logger.debug("reconnects")
logger.debug("node order: {}".format(names))
logger.debug("rwaits: {}".format(rwaits))
def testNodeConnection(allPluginsPath, tdirAndLooper, nodeReg):
console = getConsole()
console.reinit(flushy=True, verbosity=console.Wordage.verbose)
tdir, looper = tdirAndLooper
names = ["Alpha", "Beta"]
logger.debug(names)
nrg = {n: nodeReg[n] for n in names}
A, B = [TestNode(name, nrg, basedirpath=tdir,
pluginPaths=allPluginsPath)
for name in names]
looper.add(A)
A.startKeySharing()
looper.runFor(4)
logger.debug("wait done")
looper.add(B)
B.startKeySharing()
looper.runFor(4)
looper.run(checkNodesConnected([A, B]))
looper.stopall()
A.start(looper.loop)
looper.runFor(4)
B.start(looper.loop)
looper.run(checkNodesConnected([A, B]))
stopNodes([A, B], looper)
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func,
looper_without_nodeset_for_func, nodeReg):
"""
The nodes Alpha and Beta know about each other so they should connect but
they should remove remote for C when it tries to connect to them
"""
looper = looper_without_nodeset_for_func
names = ["Alpha", "Beta"]
nrg = {n: nodeReg[n] for n in names}
initLocalKeys(tdir_for_func, tconf_for_func, nrg)
logger.debug(names)
nodes = []
for name in names:
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
node = TestNode(name,
nrg,
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
nodes.append(node)
for node in nodes:
tellKeysToOthers(node, nodes)
A, B = nodes
for node in nodes:
looper.add(node)
looper.run(checkNodesConnected(nodes))
name = "Gamma"
initLocalKeys(tdir_for_func, tconf_for_func, {name: nodeReg[name]})
config_helper = PNodeConfigHelper(name,
tconf_for_func,
chroot=tdir_for_func)
C = TestNode(name, {
**nrg,
**{
name: nodeReg[name]
}
},
config_helper=config_helper,
config=tconf_for_func,
pluginPaths=allPluginsPath)
for node in nodes:
tellKeysToOthers(node, [
C,
])
looper.add(C)
looper.runFor(5)
stopNodes([
C,
], looper)
timeout = waits.expectedPoolInterconnectionTime(len(nodeReg))
stopNodes([A, B], looper)