def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
def test_node_catchup_after_restart_with_txns(
sdk_new_node_caught_up, txnPoolNodeSet, tdir, tconf,
sdk_node_set_with_node_added_after_some_txns, allPluginsPath):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \
sdk_node_set_with_node_added_after_some_txns
logger.debug("Stopping node {} with pool ledger size {}".format(
new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(new_node)
# for n in txnPoolNodeSet[:4]:
# for r in n.nodestack.remotes.values():
# if r.name == newNode.name:
# r.removeStaleCorrespondents()
# looper.run(eventually(checkNodeDisconnectedFrom, newNode.name,
# txnPoolNodeSet[:4], retryWait=1, timeout=5))
# TODO: Check if the node has really stopped processing requests?
logger.debug("Sending requests")
more_requests = 5
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
new_steward_wallet_handle, more_requests)
logger.debug("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir)
newNode = TestNode(new_node.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
# Make sure ledger is not synced initially
check_ledger_state(newNode, DOMAIN_LEDGER_ID, LedgerState.not_synced)
# Delay catchup reply processing so LedgerState does not change
# TODO fix delay, sometimes it's not enough and lower 'check_ledger_state'
# fails because newNode's domain ledger state is 'synced'
delay_catchup_reply = 10
newNode.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(
eventually(check_ledger_state,
newNode,
DOMAIN_LEDGER_ID,
LedgerState.syncing,
retryWait=.5,
timeout=5))
confused_node = txnPoolNodeSet[0]
new_node_ledger = newNode.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID]
cp = new_node_ledger.catchUpTill
start, end = cp.seqNoStart, cp.seqNoEnd
cons_proof = confused_node.ledgerManager._buildConsistencyProof(
DOMAIN_LEDGER_ID, start, end)
bad_send_time = None
def chk():
nonlocal bad_send_time
entries = newNode.ledgerManager.spylog.getAll(
newNode.ledgerManager.canProcessConsistencyProof.__name__)
for entry in entries:
# `canProcessConsistencyProof` should return False after `syncing_time`
if entry.result == False and entry.starttime > bad_send_time:
return
assert False
def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
send_and_chk(LedgerState.syncing)
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper,
newNode,
*txnPoolNodeSet[:-1],
customTimeout=timeout)
assert new_node_ledger.num_txns_caught_up == more_requests
send_and_chk(LedgerState.synced)
def test_node_catchup_after_restart_with_txns(
sdk_new_node_caught_up,
txnPoolNodeSet,
tdir,
tconf,
sdk_node_set_with_node_added_after_some_txns,
allPluginsPath):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \
sdk_node_set_with_node_added_after_some_txns
logger.debug("Stopping node {} with pool ledger size {}".
format(new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(new_node)
# for n in txnPoolNodeSet[:4]:
# for r in n.nodestack.remotes.values():
# if r.name == newNode.name:
# r.removeStaleCorrespondents()
# looper.run(eventually(checkNodeDisconnectedFrom, newNode.name,
# txnPoolNodeSet[:4], retryWait=1, timeout=5))
# TODO: Check if the node has really stopped processing requests?
logger.debug("Sending requests")
more_requests = 5
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
new_steward_wallet_handle, more_requests)
logger.debug("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir)
newNode = TestNode(
new_node.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
# Make sure ledger is not synced initially
check_ledger_state(newNode, DOMAIN_LEDGER_ID, LedgerState.not_synced)
# Delay catchup reply processing so LedgerState does not change
# TODO fix delay, sometimes it's not enough and lower 'check_ledger_state'
# fails because newNode's domain ledger state is 'synced'
delay_catchup_reply = 10
newNode.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(eventually(check_ledger_state, newNode, DOMAIN_LEDGER_ID,
LedgerState.syncing, retryWait=.5, timeout=5))
confused_node = txnPoolNodeSet[0]
new_node_ledger = newNode.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID]
cp = new_node_ledger.catchUpTill
start, end = cp.seqNoStart, cp.seqNoEnd
cons_proof = confused_node.ledgerManager._buildConsistencyProof(
DOMAIN_LEDGER_ID, start, end)
bad_send_time = None
def chk():
nonlocal bad_send_time
entries = newNode.ledgerManager.spylog.getAll(
newNode.ledgerManager.canProcessConsistencyProof.__name__)
for entry in entries:
# `canProcessConsistencyProof` should return False after `syncing_time`
if entry.result == False and entry.starttime > bad_send_time:
return
assert False
def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
send_and_chk(LedgerState.syncing)
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1],
customTimeout=timeout)
send_and_chk(LedgerState.synced)
