def test_no_propagate_request_on_different_last_ordered_on_master_before_vc(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary after next view change) are already ordered transaction on master
and slow_nodes are not. Check ordering on slow_nodes.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
master_instance = txnPoolNodeSet[0].master_replica.instId
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, cDelay()):
# send one request
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_for_slow = slow_nodes[0].master_replica.last_ordered_3pc
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered, fast_nodes, master_instance,
(old_view_no, old_last_ordered[1] + 1)))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
replies = sdk_get_replies(looper, requests)
for reply in replies:
sdk_check_reply(reply)
check_last_ordered(slow_nodes, master_instance,
(old_view_no, last_ordered_for_slow[1] + 1))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_view_change_during_unstash(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, tconf):
slow_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
slow_stasher = slow_node.nodeIbStasher
other_stashers = [n.nodeIbStasher for n in other_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
# Preload nodes with some transactions
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
for node in txnPoolNodeSet:
assert node.master_replica.last_ordered_3pc == (0, 1)
# Prevent ordering of some requests
start_delaying(all_stashers, delay_3pc(after=7, msgs=(Prepare, Commit)))
# Stop ordering on slow node and send requests
slow_node_after_5 = start_delaying(slow_stasher,
delay_3pc(after=5, msgs=Commit))
slow_node_until_5 = start_delaying(slow_stasher, delay_3pc(after=0))
reqs_view_0 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 8)
# Make pool order first 2 batches and pause
pool_after_3 = start_delaying(other_stashers, delay_3pc(after=3))
looper.run(eventually(check_nodes_ordered_till, other_nodes, 0, 3))
# Start catchup, continue ordering everywhere (except two last batches on slow node)
with delay_rules(slow_stasher, cr_delay()):
slow_node._do_start_catchup(just_started=False)
looper.run(eventually(check_catchup_is_started, slow_node))
stop_delaying_and_process(pool_after_3)
looper.run(eventually(check_nodes_ordered_till, other_nodes, 0, 7))
# Finish catchup and continue processing on slow node
looper.run(eventually(check_catchup_is_finished, slow_node))
stop_delaying_and_process(slow_node_until_5)
looper.run(eventually(check_nodes_ordered_till, [slow_node], 0, 5))
# Start view change and allow slow node to get remaining commits
with delay_rules(all_stashers, icDelay()):
for node in txnPoolNodeSet:
node.view_changer.on_master_degradation()
looper.runFor(0.1)
stop_delaying_and_process(slow_node_after_5)
# Ensure that expected number of requests was ordered
replies = sdk_get_replies(looper, reqs_view_0)
for rep in replies[:6]:
sdk_check_reply(rep)
# Ensure that everything is ok
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def sdk_get_and_check_multiply_replies(looper, request_couple):
rets = []
for req_res in sdk_get_replies(looper, [
request_couple,
]):
req, responses = req_res
if not isinstance(responses, ErrorCode) and "op" not in responses:
for node_resp in responses.values():
sdk_check_reply((req, json.loads(node_resp)))
else:
sdk_check_reply(req_res)
rets.append(req_res)
return rets[0]
def test_no_propagate_request_on_different_last_ordered_on_master_before_vc(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary after next view change) are already ordered transaction on master
and slow_nodes are not. Check ordering on slow_nodes.'''
global batches_count
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
batches_count += 1
master_instance = txnPoolNodeSet[0].master_replica.instId
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
assert batches_count == old_last_ordered[1]
with delay_rules(nodes_stashers, cDelay()):
# send one request
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
batches_count += 1
last_ordered_for_slow = slow_nodes[0].master_replica.last_ordered_3pc
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered, fast_nodes, master_instance,
(old_view_no, batches_count)))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=60)
batches_count += 1
replies = sdk_get_replies(looper, requests)
for reply in replies:
sdk_check_reply(reply)
# a new primary will send a PrePrepare for the new view
looper.run(
eventually(check_last_ordered, txnPoolNodeSet, master_instance,
(old_view_no + 1, batches_count)))
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def testOnlyAStewardCanAddAnotherSteward(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward,
alias='testSteward' + randomString(3), role=STEWARD_STRING)
seed = randomString(32)
wh, _ = sdk_wallet_client
nym_request, steward_did = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_client, seed,
'testSteward2', 'STEWARD'))
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client,
sdk_pool_handle, nym_request)
total_timeout = sdk_eval_timeout(1, len(txnPoolNodeSet))
request_couple = sdk_get_replies(looper, [request_couple], total_timeout)[0]
with pytest.raises(RequestRejectedException) as e:
sdk_check_reply(request_couple)
assert 'Only Steward is allowed to do these transactions' in e._excinfo[1].args[0]
def test_no_propagate_request_on_different_last_ordered_on_master_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary after next view change) are already ordered transaction on master
and slow_nodes are not. Check ordering on slow_nodes.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
master_instance = txnPoolNodeSet[0].master_replica.instId
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, cDelay()):
# send one request
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_for_slow = slow_nodes[0].master_replica.last_ordered_3pc
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered,
fast_nodes,
master_instance,
(old_view_no, old_last_ordered[1] + 1)))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
replies = sdk_get_replies(looper, requests)
for reply in replies:
sdk_check_reply(reply)
check_last_ordered(slow_nodes,
master_instance,
(old_view_no, last_ordered_for_slow[1] + 1))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on(
looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
timeout = sdk_eval_timeout(1, len(txnPoolNodeSet))
nodes = txnPoolNodeSet
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
stop_node(nodes[0], looper, nodes)
waitForViewChange(looper, nodes[1:], expectedViewNo=1)
ensureElectionsDone(looper,
nodes[1:],
numInstances=getRequiredInstances(nodeCount))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
stop_node(nodes[1], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[2:], expectedViewNo=1)
sdk_reqs3 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
with pytest.raises(TimeoutError):
req_res = sdk_get_replies(looper, sdk_reqs3, timeout=timeout)
sdk_check_reply(req_res[0])
stop_node(nodes[2], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[3:], expectedViewNo=1)
sdk_reqs4 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
with pytest.raises(TimeoutError):
req_res = sdk_get_replies(looper, sdk_reqs4, timeout=timeout)
sdk_check_reply(req_res[0])
nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir,
allPluginsPath)
looper.runFor(waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[3:], expectedViewNo=1)
sdk_reqs5 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
with pytest.raises(TimeoutError):
req_res = sdk_get_replies(looper, sdk_reqs5, timeout=timeout)
sdk_check_reply(req_res[0])
nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir,
allPluginsPath)
ensureElectionsDone(looper,
nodes[1:],
numInstances=getRequiredInstances(nodeCount))
checkViewNoForNodes(nodes[1:], expectedViewNo=1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir,
allPluginsPath)
ensureElectionsDone(looper,
nodes,
numInstances=getRequiredInstances(nodeCount))
checkViewNoForNodes(nodes, expectedViewNo=1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on(
looper, allPluginsPath, tdir, tconf,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
timeout = sdk_eval_timeout(1, len(txnPoolNodeSet))
nodes = txnPoolNodeSet
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[0], looper, nodes)
waitForViewChange(looper, nodes[1:], expectedViewNo=1)
ensureElectionsDone(looper, nodes[1:],
instances_list=range(getRequiredInstances(nodeCount)))
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[1], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[2:], expectedViewNo=1)
sdk_reqs3 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs3, timeout=timeout)
sdk_check_reply(req_res[0])
stop_node(nodes[2], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[3:], expectedViewNo=1)
sdk_reqs4 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs4, timeout=timeout)
sdk_check_reply(req_res[0])
nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath)
looper.runFor(waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[3:], expectedViewNo=1)
sdk_reqs5 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs5, timeout=timeout)
sdk_check_reply(req_res[0])
nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes[1:],
instances_list=range(getRequiredInstances(nodeCount)),
customTimeout=60)
checkViewNoForNodes(nodes[1:], expectedViewNo=1)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes,
instances_list=range(getRequiredInstances(nodeCount)),
customTimeout=60)
checkViewNoForNodes(nodes, expectedViewNo=1)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on(
looper, allPluginsPath, tdir, tconf,
txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
nodes = txnPoolNodeSet
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[4], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:4], expectedViewNo=0)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[3], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:3], expectedViewNo=0)
sdk_reqs3 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs3)
sdk_check_reply(req_res[0])
stop_node(nodes[2], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:2], expectedViewNo=0)
sdk_reqs4 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs4)
sdk_check_reply(req_res[0])
nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath)
looper.runFor(waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0)
sdk_reqs5 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs5)
sdk_check_reply(req_res[0])
nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes[:2] + nodes[3:],
numInstances=getRequiredInstances(nodeCount))
checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0)
sdk_reqs6 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
sdk_get_replies(looper, [sdk_reqs3[0], sdk_reqs4[0], sdk_reqs5[0], sdk_reqs6[0]])
nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes,
numInstances=getRequiredInstances(nodeCount))
checkViewNoForNodes(nodes, expectedViewNo=0)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
def succeeded(req_resp):
try:
sdk_check_reply(req_resp)
return True
except Exception:
return False
def test_already_processed_requests(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
"""
Client re-sending request and checking that nodes picked the reply from
ledger and did not process the request again
"""
def get_method_call_count(method):
counts = set()
for node in txnPoolNodeSet:
c = node.spylog.count(method)
counts.add(c)
assert len(counts) == 1
return counts.pop()
def get_getReplyFromLedger_call_count():
return get_method_call_count(
next(iter(txnPoolNodeSet)).getReplyFromLedger)
def get_recordAndPropagate_call_count():
return get_method_call_count(
next(iter(txnPoolNodeSet)).recordAndPropagate)
def get_last_returned_val():
rvs = []
for node in txnPoolNodeSet:
rv = getAllReturnVals(node, node.getReplyFromLedger)
rvs.append(rv[0])
# All items are same in the list
assert rvs.count(rvs[0]) == len(txnPoolNodeSet)
return rvs[0]
rlc1 = get_getReplyFromLedger_call_count()
rpc1 = get_recordAndPropagate_call_count()
# Request which will be send twice
reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
# Send, check and getting reply from first request
sdk_reqs = sdk_send_signed_requests(sdk_pool_handle, reqs)
total_timeout = sdk_eval_timeout(len(sdk_reqs), len(txnPoolNodeSet))
request1 = sdk_get_replies(looper, sdk_reqs, timeout=total_timeout)
for req_res in request1:
sdk_check_reply(req_res)
first_req_id = request1[0][0]['reqId']
rlc2 = get_getReplyFromLedger_call_count()
rpc2 = get_recordAndPropagate_call_count()
assert rlc2 - rlc1 == 1 # getReplyFromLedger was called
assert rpc2 - rpc1 == 1 # recordAndPropagate was called
r1 = get_last_returned_val()
assert r1 is None # getReplyFromLedger returned None since had not seen request
# Request which we will send only once
request2 = sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 1)
second_req_id = request2[0][0]['reqId']
assert second_req_id != first_req_id
rlc3 = get_getReplyFromLedger_call_count()
rpc3 = get_recordAndPropagate_call_count()
assert rlc3 - rlc2 == 1 # getReplyFromLedger was called again
assert rpc3 - rpc2 == 1 # recordAndPropagate was called again
r2 = get_last_returned_val()
assert r2 is None # getReplyFromLedger returned None since had not seen request
# Reply for the first request, which is going to be sent again
rep1 = request1[0][1]['result']
# Client re-sending first request
request3 = sdk_send_signed_requests(sdk_pool_handle, reqs)
total_timeout = sdk_eval_timeout(len(request3), len(txnPoolNodeSet))
request3 = sdk_get_replies(looper, request3, timeout=total_timeout)
third_req_id = request3[0][0]['reqId']
assert third_req_id == first_req_id
rlc4 = get_getReplyFromLedger_call_count()
rpc4 = get_recordAndPropagate_call_count()
assert rlc4 - rlc3 == 1 # getReplyFromLedger was called again
assert rpc4 - rpc3 == 0 # recordAndPropagate was not called
r3 = get_last_returned_val()
# getReplyFromLedger did not return None this time since had seen request
assert r3 is not None
rep3 = request3[0][1]['result']
# Since txnTime is not stored in ledger and reading from ledger return
# all possible fields from transactions
rep3 = {k: v for k, v in rep3.items() if v is not None}
rep1 = {k: v for k, v in rep1.items() if k in rep3}
assert rep3 == rep1 # The reply client got is same as the previous one
def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on(
looper, allPluginsPath, tdir, tconf,
txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
nodes = txnPoolNodeSet
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[4], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:4], expectedViewNo=0)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stop_node(nodes[3], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:3], expectedViewNo=0)
sdk_reqs3 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs3)
sdk_check_reply(req_res[0])
stop_node(nodes[2], looper, nodes)
looper.runFor(tconf.ToleratePrimaryDisconnection +
waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:2], expectedViewNo=0)
sdk_reqs4 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs4)
sdk_check_reply(req_res[0])
nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath)
looper.runFor(waits.expectedPoolElectionTimeout(len(nodes)))
checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0)
sdk_reqs5 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
with pytest.raises(PoolLedgerTimeoutException):
req_res = sdk_get_replies(looper, sdk_reqs5)
sdk_check_reply(req_res[0])
nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes[:2] + nodes[3:],
instances_list=range(getRequiredInstances(nodeCount)))
checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0)
sdk_reqs6 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
sdk_get_replies(looper, sdk_reqs6)
nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, nodes,
instances_list=range(getRequiredInstances(nodeCount)))
checkViewNoForNodes(nodes, expectedViewNo=0)
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
def test_already_processed_requests(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Client re-sending request and checking that nodes picked the reply from
ledger and did not process the request again
"""
def get_method_call_count(method):
counts = set()
for node in txnPoolNodeSet:
c = node.spylog.count(method)
counts.add(c)
assert len(counts) == 1
return counts.pop()
def get_getReplyFromLedgerForRequest_call_count():
return get_method_call_count(
next(iter(txnPoolNodeSet)).getReplyFromLedgerForRequest)
def get_recordAndPropagate_call_count():
return get_method_call_count(
next(iter(txnPoolNodeSet)).recordAndPropagate)
def get_last_returned_val():
rvs = []
for node in txnPoolNodeSet:
rv = getAllReturnVals(node, node.getReplyFromLedgerForRequest)
rvs.append(rv[0])
# All items are same in the list
assert rvs.count(rvs[0]) == len(txnPoolNodeSet)
return rvs[0]
rlc1 = get_getReplyFromLedgerForRequest_call_count()
rpc1 = get_recordAndPropagate_call_count()
# Request which will be send twice
reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
# Send, check and getting reply from first request
sdk_reqs = sdk_send_signed_requests(sdk_pool_handle, reqs)
total_timeout = sdk_eval_timeout(len(sdk_reqs), len(txnPoolNodeSet))
request1 = sdk_get_replies(looper, sdk_reqs, timeout=total_timeout)
for req_res in request1:
sdk_check_reply(req_res)
first_req_id = request1[0][0]['reqId']
rlc2 = get_getReplyFromLedgerForRequest_call_count()
rpc2 = get_recordAndPropagate_call_count()
assert rlc2 - rlc1 == 1 # getReplyFromLedgerForRequest was called
assert rpc2 - rpc1 == 1 # recordAndPropagate was called
r1 = get_last_returned_val()
assert r1 is None # getReplyFromLedgerForRequest returned None since had not seen request
# Request which we will send only once
request2 = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
second_req_id = request2[0][0]['reqId']
assert second_req_id != first_req_id
rlc3 = get_getReplyFromLedgerForRequest_call_count()
rpc3 = get_recordAndPropagate_call_count()
assert rlc3 - rlc2 == 1 # getReplyFromLedgerForRequest was called again
assert rpc3 - rpc2 == 1 # recordAndPropagate was called again
r2 = get_last_returned_val()
assert r2 is None # getReplyFromLedgerForRequest returned None since had not seen request
# Reply for the first request, which is going to be sent again
rep1 = request1[0][1]['result']
# Client re-sending first request
request3 = sdk_send_signed_requests(sdk_pool_handle, reqs)
total_timeout = sdk_eval_timeout(len(request3), len(txnPoolNodeSet))
request3 = sdk_get_replies(looper, request3, timeout=total_timeout)
third_req_id = request3[0][0]['reqId']
assert third_req_id == first_req_id
rlc4 = get_getReplyFromLedgerForRequest_call_count()
rpc4 = get_recordAndPropagate_call_count()
assert rlc4 - rlc3 == 1 # getReplyFromLedgerForRequest was called again
assert rpc4 - rpc3 == 0 # recordAndPropagate was not called
r3 = get_last_returned_val()
# getReplyFromLedgerForRequest did not return None this time since had seen request
assert r3 is not None
rep3 = request3[0][1]['result']
# Since txnTime is not stored in ledger and reading from ledger return
# all possible fields from transactions
rep3 = {k: v for k, v in rep3.items() if v is not None}
rep1 = {k: v for k, v in rep1.items() if k in rep3}
assert rep3 == rep1 # The reply client got is same as the previous one
