def test_state_proof_for_get_fees(helpers, nodeSetWithIntegratedTokenPlugin):
fees = {NYM_FEES_ALIAS: 5}
with delay_rules(
[n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
with pytest.raises(PoolLedgerTimeoutException):
helpers.general.do_get_fees()
helpers.general.do_set_fees(fees)
with delay_rules(
[n.nodeIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
resp = helpers.general.do_get_fees()
assert resp.get(STATE_PROOF, False)
assert fees == resp[FEES]
def nodeSetWithOneRespondingNode(request, nodeSetWithIntegratedTokenPlugin, make_tokens, last_txn):
if request.param == 'all_responding':
yield nodeSetWithIntegratedTokenPlugin
else:
stashers = [node.clientIbStasher for node in nodeSetWithIntegratedTokenPlugin[1:]]
with delay_rules(stashers, req_delay()):
yield nodeSetWithIntegratedTokenPlugin
def test_belated_request_not_processed_if_already_in_3pc_process(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
delta.clientIbStasher.delay(req_delay(300))
for node in txnPoolNodeSet:
node.nodeIbStasher.delay(cDelay(300))
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_signed_requests(sdk_pool_handle, one_req)
looper.runFor(
waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
delta.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(
waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
node.nodeIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def test_forced_upgrade_handled_once_if_ordered_and_then_request_received(
looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled one time in case
the node commits the transaction to the ledger and only after that receives
the request directly from the client
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
slow_node.clientIbStasher.delay(req_delay())
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.run(
eventually(checkUpgradeScheduled, [slow_node],
validUpgradeExpForceTrue[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
slow_node.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedUpgradeScheduled())
checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION])
assert count_action_log_package(list(slow_node.upgrader._actionLog),
validUpgradeExpForceTrue['package']) == 1
assert slow_node.upgrader._actionLog.last_event.ev_type == UpgradeLog.Events.scheduled
def test_belated_request_not_processed_if_already_in_3pc_process(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
delta.clientIbStasher.delay(req_delay(300))
for node in txnPoolNodeSet:
node.nodeIbStasher.delay(cDelay(300))
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_signed_requests(sdk_pool_handle, one_req)
looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
delta.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
node.nodeIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def nodeSetAlwaysResponding(request, txnPoolNodeSet, transactions):
if request.param == 'all_responding':
yield txnPoolNodeSet
else:
stashers = [node.clientIbStasher for node in txnPoolNodeSet[1:]]
with delay_rules(stashers, req_delay()):
yield txnPoolNodeSet
def test_node_handles_forced_upgrade_on_propagate(looper, nodeSet,
sdk_pool_handle,
sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled immediately when
the node receives it in a PROPAGATE from any other node
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
# Stash all except PROPAGATEs from Gamma
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Alpha'))
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta'))
slow_node.nodeIbStasher.delay(ppDelay())
slow_node.nodeIbStasher.delay(pDelay())
slow_node.nodeIbStasher.delay(cDelay())
sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.run(
eventually(checkUpgradeScheduled, [slow_node],
validUpgradeExpForceTrue[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def nodeSetWithTaa(request, nodeSetWithTaaAlwaysResponding):
if request.param == 'all_responding':
yield nodeSetWithTaaAlwaysResponding
else:
stashers = [node.clientIbStasher for node in nodeSetWithTaaAlwaysResponding[1:]]
with delay_rules(stashers, req_delay()):
yield nodeSetWithTaaAlwaysResponding
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle, teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay())
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(msg_rep_delay(1000, [
PROPAGATE,
]))
old_count_resp = count_requested_preprepare_resp(slow_node)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=4,
num_batches=2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def nodeSetWithOneNodeResponding(nodeSet):
# the order of nodes the client sends requests to is [Alpha, Beta, Gamma, Delta]
# delay all requests to Beta, Gamma and Delta
# we expect that it's sufficient for the client to get Reply from Alpha only
# as for write requests, we can send it to 1 node only, and it will be propagated to others
for node in nodeSet[1:]:
node.clientIbStasher.delay(req_delay())
return nodeSet
def setup(txnPoolNodeSet):
faulty_node = txnPoolNodeSet[-1]
# Set quorum a bit more, so that faulty_node will request propagates
faulty_node.quorums.propagate = Quorum(3)
# do not receive requests and Propagates so that Propagates will be requested
faulty_node.clientIbStasher.delay(req_delay())
faulty_node.nodeIbStasher.delay(ppgDelay())
return faulty_node
def test_state_proof_for_get_fees(helpers, nodeSetWithIntegratedTokenPlugin,
looper, sdk_pool_handle, sdk_wallet_trustee):
# make sure that config ledger is BLS signed by sending a txn to config ledger
send_and_check_auth_rule(looper, sdk_pool_handle, sdk_wallet_trustee)
with delay_rules(
[n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
resp = helpers.general.do_get_fees()
assert resp.get(STATE_PROOF, False)
assert {} == resp[FEES]
fees = {NYM_FEES_ALIAS: 5}
helpers.general.do_set_fees(fees)
with delay_rules(
[n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
resp = helpers.general.do_get_fees()
assert resp.get(STATE_PROOF, False)
assert fees == resp[FEES]
def test_belated_request_not_processed_if_already_ordered(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
delta.clientIbStasher.delay(req_delay(300))
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle)
delta.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def testOneNodeAltersAClientRequest(looper,
txnPoolNodeSet,
evilAlpha,
sdk_pool_handle,
sdk_wallet_client):
"""Malicious Alpha node sends incorrect propagate. This test check that
nodes raise InsufficientCorrectSignatures in validate this propagate"""
# TODO: This test is throwing a `indy.error.PoolLedgerTerminated` exception
# This is probably happening because a request is sent and the pool is terminated before the reply is processed
alpha = txnPoolNodeSet[0]
goodNodes = list(txnPoolNodeSet)
goodNodes.remove(alpha)
# delay incoming client messages for good nodes by 250 milliseconds
# this gives Alpha a chance to send a propagate message
for n in goodNodes: # type: TestNode
n.nodeIbStasher.delay(ppDelay(sys.maxsize))
n.nodeIbStasher.delay(req_delay(sys.maxsize))
pastNodes = []
request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
sent1 = sdk_json_to_request_object(request_couple_json[0][0])
checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes)
def check():
for node in goodNodes:
if node not in pastNodes:
# ensure the nodes are suspicious of Alpha
params = node.spylog.getLastParams(TestNode.reportSuspiciousNode)
frm = params["nodeName"]
reason = params["reason"]
assert frm == 'Alpha'
invalid_signatures = 'did={}, signature={}'.format(sent1.identifier, sent1.signature)
assert reason == InsufficientCorrectSignatures.reason.format(1, 0, 1, invalid_signatures)
# ensure Alpha's propagates were ignored by the other nodes
key = sent1.digest
props = node.requests[key].propagates
assert 'Alpha' not in props
for good in goodNodes:
assert good.name in props
pastNodes.append(node)
for node in goodNodes:
node.nodeIbStasher.resetDelays()
timeout = waits.expectedClientRequestPropagationTime(len(txnPoolNodeSet))
looper.run(eventually(check, retryWait=1, timeout=timeout))
def test_belated_request_not_processed_if_already_ordered(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
delta.clientIbStasher.delay(req_delay(300))
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle)
delta.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def testOneNodeAltersAClientRequest(looper,
txnPoolNodeSet,
evilAlpha,
sdk_pool_handle,
sdk_wallet_client):
"""Malicious Alpha node sends incorrect propagate. This test check that
nodes raise InsufficientCorrectSignatures in validate this propagate"""
alpha = txnPoolNodeSet[0]
goodNodes = list(txnPoolNodeSet)
goodNodes.remove(alpha)
# delay incoming client messages for good nodes by 250 milliseconds
# this gives Alpha a chance to send a propagate message
for n in goodNodes: # type: TestNode
n.nodeIbStasher.delay(ppDelay(sys.maxsize))
n.nodeIbStasher.delay(req_delay(sys.maxsize))
pastNodes = []
request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
sent1 = sdk_json_to_request_object(request_couple_json[0][0])
checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes)
def check():
for node in goodNodes:
if node not in pastNodes:
# ensure the nodes are suspicious of Alpha
params = node.spylog.getLastParams(TestNode.reportSuspiciousNode)
frm = params["nodeName"]
reason = params["reason"]
assert frm == 'Alpha'
assert reason == InsufficientCorrectSignatures.reason.format(0, 1)
# ensure Alpha's propagates were ignored by the other nodes
key = sent1.digest
props = node.requests[key].propagates
assert 'Alpha' not in props
for good in goodNodes:
assert good.name in props
pastNodes.append(node)
for node in goodNodes:
node.nodeIbStasher.resetDelays()
timeout = waits.expectedClientRequestPropagationTime(len(txnPoolNodeSet))
looper.run(eventually(check, retryWait=1, timeout=timeout))
def setup(request, txnPoolNodeSet):
# Test once when client request is received and once when not received
# Choosing a faulty node which is primary in neither instance, this helps
# in the that same PROPAGATEs are not requested again by the node
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node
if request.param == 'client_requests':
# Long delay in PROPAGATEs
faulty_node.nodeIbStasher.delay(ppgDelay(90))
return faulty_node, True
if request.param == 'no_client_requests':
# Long delay in PROPAGATEs
faulty_node.nodeIbStasher.delay(ppgDelay(90))
# Long delay in Client Requests
faulty_node.clientIbStasher.delay(req_delay(90))
return faulty_node, False
def setup(request, txnPoolNodeSet):
# Test once when client request is received and once when not received
# Choosing a faulty node which is primary in neither instance, this helps
# in the that same PROPAGATEs are not requested again by the node
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node
if request.param == 'client_requests':
# Long delay in PROPAGATEs
faulty_node.nodeIbStasher.delay(ppgDelay(90))
return faulty_node, True
if request.param == 'no_client_requests':
# Long delay in PROPAGATEs
faulty_node.nodeIbStasher.delay(ppgDelay(90))
# Long delay in Client Requests
faulty_node.clientIbStasher.delay(req_delay(90))
return faulty_node, False
def test_forced_upgrade_handled_once_if_request_received_after_propagate(
looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled one time in case
the node commits the transaction to the ledger but during the 3PC-process
receives the request directly from the client after a PROPAGATE from some
other node
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta'))
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Gamma'))
original_process_propagate = slow_node.nodeMsgRouter.routes[Propagate]
original_process_request = slow_node.clientMsgRouter.routes[Request]
def patched_process_propagate(msg: Propagate, frm: str):
original_process_propagate(msg, frm)
slow_node.clientIbStasher.reset_delays_and_process_delayeds()
slow_node.nodeMsgRouter.routes[Propagate] = original_process_propagate
def patched_process_request(request: Request, frm: str):
original_process_request(request, frm)
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
slow_node.clientMsgRouter.routes[Request] = original_process_request
slow_node.nodeMsgRouter.routes[Propagate] = patched_process_propagate
slow_node.clientMsgRouter.routes[Request] = patched_process_request
init_len = len(list(slow_node.upgrader._actionLog))
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.runFor(waits.expectedUpgradeScheduled())
checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION])
if init_len == 0:
# first upgrade - should be only one scheduled
assert len(list(slow_node.upgrader._actionLog)) == 1
else:
# one upgrade were already scheduled. we should cancel it and schedule new one
# so action log should be increased by 2
assert len(list(slow_node.upgrader._actionLog)) == init_len + 2
assert slow_node.upgrader._actionLog.lastEvent[1] == UpgradeLog.SCHEDULED
def setup(txnPoolNodeSet):
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node
faulty_node.nodeIbStasher.delay(ppgDelay())
faulty_node.clientIbStasher.delay(req_delay())
return faulty_node
