def testTestNodeDelay(tdir_for_func):
nodeNames = {"testA", "testB"}
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
nodeA = nodes.getNode("testA")
nodeB = nodes.getNode("testB")
with Looper(nodes) as looper:
looper.run(checkNodesConnected(nodes))
# send one message, without delay
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
# set delay, then send another message
# and find that it doesn't arrive
delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime()
nodeB.nodeIbStasher.delay(
delayerMsgTuple(delay, TestMsg, nodeA.name)
)
with pytest.raises(AssertionError):
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
# but then find that it arrives after the delay
# duration has passed
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB,
customTimeout=timeout))
# reset the delay, and find another message comes quickly
nodeB.nodeIbStasher.reset_delays_and_process_delayeds()
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
def testStewardSuspendsNode(looper, txnPoolNodeSet,
tdir, tconf,
sdk_pool_handle,
sdk_wallet_steward,
sdk_node_theta_added,
poolTxnStewardData,
allPluginsPath):
new_steward_wallet, new_node = sdk_node_theta_added
demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
# Check suspended node does not exist in any nodeReg or remotes of
# nodes or clients
txnPoolNodeSet = txnPoolNodeSet[:-1]
for node in txnPoolNodeSet:
looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name))
# Check that a node does not connect to the suspended
# node
sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
with pytest.raises(RemoteNotFound):
looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node))
new_node.stop()
looper.removeProdable(new_node)
# Check that a node whose suspension is revoked can reconnect to other
# nodes and clients can also connect to that node
promote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
nodeTheta = start_stopped_node(new_node, looper, tconf,
tdir, allPluginsPath,
delay_instance_change_msgs=False)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testMsgSendingTime(pool):
nodes = pool.nodeset
msg = randomMsg()
timeout = waits.expectedNodeStartUpTimeout()
pool.looper.run(
sendMessageAndCheckDelivery(nodes[0],
nodes[1],
msg,
customTimeout=timeout))
def testMsgSendingTime(pool, nodeReg):
nodeNames = list(nodeReg.keys())
msg = randomMsg()
timeout = waits.expectedNodeStartUpTimeout()
pool.looper.run(
sendMessageAndCheckDelivery(pool.nodeset,
nodeNames[0],
nodeNames[1],
msg,
customTimeout=timeout))
def test_steward_suspends_node_and_promote_with_new_ha(
looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle,
sdk_wallet_steward, sdk_node_theta_added, poolTxnStewardData,
allPluginsPath):
new_steward_wallet, new_node = sdk_node_theta_added
looper.run(checkNodesConnected(txnPoolNodeSet + [new_node]))
demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
# Check suspended node does not exist in any nodeReg or remotes of
# nodes or clients
txnPoolNodeSet = txnPoolNodeSet[:-1]
for node in txnPoolNodeSet:
looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name))
# Check that a node does not connect to the suspended
# node
sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet,
sdk_pool_handle)
with pytest.raises(RemoteNotFound):
looper.loop.run_until_complete(
sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node))
new_node.stop()
looper.removeProdable(new_node)
# Check that a node whose suspension is revoked can reconnect to other
# nodes and clients can also connect to that node
node_ha, client_ha = genHa(2)
node_nym = hexToFriendly(new_node.nodestack.verhex)
sdk_send_update_node(looper,
new_steward_wallet,
sdk_pool_handle,
node_nym,
new_node.name,
node_ha.host,
node_ha.port,
client_ha.host,
client_ha.port,
services=[VALIDATOR])
new_node.nodestack.ha = node_ha
new_node.clientstack.ha = client_ha
nodeTheta = start_stopped_node(new_node,
looper,
tconf,
tdir,
allPluginsPath,
delay_instance_change_msgs=False)
assert all(node.nodestack.remotes[new_node.name].ha == node_ha
for node in txnPoolNodeSet)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def testTestNodeDelay(looper, txnPoolNodeSet):
looper.run(checkNodesConnected(txnPoolNodeSet))
nodeA = txnPoolNodeSet[0]
nodeB = txnPoolNodeSet[1]
# send one message, without delay
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
# set delay, then send another message
# and find that it doesn't arrive
delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime()
nodeB.nodeIbStasher.delay(delayerMsgTuple(delay, TestMsg, nodeA.name))
with pytest.raises(AssertionError):
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
# but then find that it arrives after the delay
# duration has passed
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB,
customTimeout=timeout))
# reset the delay, and find another message comes quickly
nodeB.nodeIbStasher.reset_delays_and_process_delayeds()
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
def test_steward_suspends_node_and_promote_with_new_ha(
looper, txnPoolNodeSet,
tdir, tconf,
sdk_pool_handle,
sdk_wallet_steward,
sdk_node_theta_added,
poolTxnStewardData,
allPluginsPath):
new_steward_wallet, new_node = sdk_node_theta_added
looper.run(checkNodesConnected(txnPoolNodeSet + [new_node]))
demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
# Check suspended node does not exist in any nodeReg or remotes of
# nodes or clients
txnPoolNodeSet = txnPoolNodeSet[:-1]
for node in txnPoolNodeSet:
looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name))
# Check that a node does not connect to the suspended
# node
sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
with pytest.raises(RemoteNotFound):
looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node))
new_node.stop()
looper.removeProdable(new_node)
# Check that a node whose suspension is revoked can reconnect to other
# nodes and clients can also connect to that node
node_ha, client_ha = genHa(2)
node_nym = hexToFriendly(new_node.nodestack.verhex)
sdk_send_update_node(looper, new_steward_wallet,
sdk_pool_handle, node_nym, new_node.name,
node_ha.host, node_ha.port,
client_ha.host, client_ha.port,
services=[VALIDATOR])
new_node.nodestack.ha = node_ha
new_node.clientstack.ha = client_ha
nodeTheta = start_stopped_node(new_node, looper, tconf,
tdir, allPluginsPath,
delay_instance_change_msgs=False)
assert all(node.nodestack.remotes[new_node.name].ha == node_ha for node in txnPoolNodeSet)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)