def test_state_proof_for_get_fees(helpers, nodeSetWithIntegratedTokenPlugin): fees = {NYM_FEES_ALIAS: 5} with delay_rules( [n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()): with pytest.raises(PoolLedgerTimeoutException): helpers.general.do_get_fees() helpers.general.do_set_fees(fees) with delay_rules( [n.nodeIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()): resp = helpers.general.do_get_fees() assert resp.get(STATE_PROOF, False) assert fees == resp[FEES]
def nodeSetWithOneRespondingNode(request, nodeSetWithIntegratedTokenPlugin, make_tokens, last_txn): if request.param == 'all_responding': yield nodeSetWithIntegratedTokenPlugin else: stashers = [node.clientIbStasher for node in nodeSetWithIntegratedTokenPlugin[1:]] with delay_rules(stashers, req_delay()): yield nodeSetWithIntegratedTokenPlugin
def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def test_forced_upgrade_handled_once_if_ordered_and_then_request_received( looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue): """ Verifies that POOL_UPGRADE force=true request is handled one time in case the node commits the transaction to the ledger and only after that receives the request directly from the client """ slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node slow_node.clientIbStasher.delay(req_delay()) sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue) looper.run( eventually(checkUpgradeScheduled, [slow_node], validUpgradeExpForceTrue[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) slow_node.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedUpgradeScheduled()) checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION]) assert count_action_log_package(list(slow_node.upgrader._actionLog), validUpgradeExpForceTrue['package']) == 1 assert slow_node.upgrader._actionLog.last_event.ev_type == UpgradeLog.Events.scheduled
def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def nodeSetAlwaysResponding(request, txnPoolNodeSet, transactions): if request.param == 'all_responding': yield txnPoolNodeSet else: stashers = [node.clientIbStasher for node in txnPoolNodeSet[1:]] with delay_rules(stashers, req_delay()): yield txnPoolNodeSet
def test_node_handles_forced_upgrade_on_propagate(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue): """ Verifies that POOL_UPGRADE force=true request is handled immediately when the node receives it in a PROPAGATE from any other node """ slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node # Stash all except PROPAGATEs from Gamma slow_node.clientIbStasher.delay(req_delay()) slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Alpha')) slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta')) slow_node.nodeIbStasher.delay(ppDelay()) slow_node.nodeIbStasher.delay(pDelay()) slow_node.nodeIbStasher.delay(cDelay()) sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue) looper.run( eventually(checkUpgradeScheduled, [slow_node], validUpgradeExpForceTrue[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def nodeSetWithTaa(request, nodeSetWithTaaAlwaysResponding): if request.param == 'all_responding': yield nodeSetWithTaaAlwaysResponding else: stashers = [node.clientIbStasher for node in nodeSetWithTaaAlwaysResponding[1:]] with delay_rules(stashers, req_delay()): yield nodeSetWithTaaAlwaysResponding
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ Node missing Propagates hence request not finalised, hence stashes PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE """ slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet) slow_node.nodeIbStasher.delay(ppgDelay()) slow_node.clientIbStasher.delay(req_delay()) slow_node.nodeIbStasher.delay(msg_rep_delay(1000, [ PROPAGATE, ])) old_count_resp = count_requested_preprepare_resp(slow_node) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=4, num_batches=2) # The slow node is behind checkNodeDataForInequality(slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp slow_node.nodeIbStasher.reset_delays_and_process_delayeds() # The slow node has processed all requests waitNodeDataEquality(looper, slow_node, *other_nodes) # PRE-PREPARE were not requested assert count_requested_preprepare_resp(slow_node) == old_count_resp
def nodeSetWithOneNodeResponding(nodeSet): # the order of nodes the client sends requests to is [Alpha, Beta, Gamma, Delta] # delay all requests to Beta, Gamma and Delta # we expect that it's sufficient for the client to get Reply from Alpha only # as for write requests, we can send it to 1 node only, and it will be propagated to others for node in nodeSet[1:]: node.clientIbStasher.delay(req_delay()) return nodeSet
def setup(txnPoolNodeSet): faulty_node = txnPoolNodeSet[-1] # Set quorum a bit more, so that faulty_node will request propagates faulty_node.quorums.propagate = Quorum(3) # do not receive requests and Propagates so that Propagates will be requested faulty_node.clientIbStasher.delay(req_delay()) faulty_node.nodeIbStasher.delay(ppgDelay()) return faulty_node
def test_state_proof_for_get_fees(helpers, nodeSetWithIntegratedTokenPlugin, looper, sdk_pool_handle, sdk_wallet_trustee): # make sure that config ledger is BLS signed by sending a txn to config ledger send_and_check_auth_rule(looper, sdk_pool_handle, sdk_wallet_trustee) with delay_rules( [n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()): resp = helpers.general.do_get_fees() assert resp.get(STATE_PROOF, False) assert {} == resp[FEES] fees = {NYM_FEES_ALIAS: 5} helpers.general.do_set_fees(fees) with delay_rules( [n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()): resp = helpers.general.do_get_fees() assert resp.get(STATE_PROOF, False) assert fees == resp[FEES]
def test_belated_request_not_processed_if_already_ordered( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def testOneNodeAltersAClientRequest(looper, txnPoolNodeSet, evilAlpha, sdk_pool_handle, sdk_wallet_client): """Malicious Alpha node sends incorrect propagate. This test check that nodes raise InsufficientCorrectSignatures in validate this propagate""" # TODO: This test is throwing a `indy.error.PoolLedgerTerminated` exception # This is probably happening because a request is sent and the pool is terminated before the reply is processed alpha = txnPoolNodeSet[0] goodNodes = list(txnPoolNodeSet) goodNodes.remove(alpha) # delay incoming client messages for good nodes by 250 milliseconds # this gives Alpha a chance to send a propagate message for n in goodNodes: # type: TestNode n.nodeIbStasher.delay(ppDelay(sys.maxsize)) n.nodeIbStasher.delay(req_delay(sys.maxsize)) pastNodes = [] request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) sent1 = sdk_json_to_request_object(request_couple_json[0][0]) checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes) def check(): for node in goodNodes: if node not in pastNodes: # ensure the nodes are suspicious of Alpha params = node.spylog.getLastParams(TestNode.reportSuspiciousNode) frm = params["nodeName"] reason = params["reason"] assert frm == 'Alpha' invalid_signatures = 'did={}, signature={}'.format(sent1.identifier, sent1.signature) assert reason == InsufficientCorrectSignatures.reason.format(1, 0, 1, invalid_signatures) # ensure Alpha's propagates were ignored by the other nodes key = sent1.digest props = node.requests[key].propagates assert 'Alpha' not in props for good in goodNodes: assert good.name in props pastNodes.append(node) for node in goodNodes: node.nodeIbStasher.resetDelays() timeout = waits.expectedClientRequestPropagationTime(len(txnPoolNodeSet)) looper.run(eventually(check, retryWait=1, timeout=timeout))
def test_belated_request_not_processed_if_already_ordered( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def testOneNodeAltersAClientRequest(looper, txnPoolNodeSet, evilAlpha, sdk_pool_handle, sdk_wallet_client): """Malicious Alpha node sends incorrect propagate. This test check that nodes raise InsufficientCorrectSignatures in validate this propagate""" alpha = txnPoolNodeSet[0] goodNodes = list(txnPoolNodeSet) goodNodes.remove(alpha) # delay incoming client messages for good nodes by 250 milliseconds # this gives Alpha a chance to send a propagate message for n in goodNodes: # type: TestNode n.nodeIbStasher.delay(ppDelay(sys.maxsize)) n.nodeIbStasher.delay(req_delay(sys.maxsize)) pastNodes = [] request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) sent1 = sdk_json_to_request_object(request_couple_json[0][0]) checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes) def check(): for node in goodNodes: if node not in pastNodes: # ensure the nodes are suspicious of Alpha params = node.spylog.getLastParams(TestNode.reportSuspiciousNode) frm = params["nodeName"] reason = params["reason"] assert frm == 'Alpha' assert reason == InsufficientCorrectSignatures.reason.format(0, 1) # ensure Alpha's propagates were ignored by the other nodes key = sent1.digest props = node.requests[key].propagates assert 'Alpha' not in props for good in goodNodes: assert good.name in props pastNodes.append(node) for node in goodNodes: node.nodeIbStasher.resetDelays() timeout = waits.expectedClientRequestPropagationTime(len(txnPoolNodeSet)) looper.run(eventually(check, retryWait=1, timeout=timeout))
def setup(request, txnPoolNodeSet): # Test once when client request is received and once when not received # Choosing a faulty node which is primary in neither instance, this helps # in the that same PROPAGATEs are not requested again by the node faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node if request.param == 'client_requests': # Long delay in PROPAGATEs faulty_node.nodeIbStasher.delay(ppgDelay(90)) return faulty_node, True if request.param == 'no_client_requests': # Long delay in PROPAGATEs faulty_node.nodeIbStasher.delay(ppgDelay(90)) # Long delay in Client Requests faulty_node.clientIbStasher.delay(req_delay(90)) return faulty_node, False
def setup(request, txnPoolNodeSet): # Test once when client request is received and once when not received # Choosing a faulty node which is primary in neither instance, this helps # in the that same PROPAGATEs are not requested again by the node faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node if request.param == 'client_requests': # Long delay in PROPAGATEs faulty_node.nodeIbStasher.delay(ppgDelay(90)) return faulty_node, True if request.param == 'no_client_requests': # Long delay in PROPAGATEs faulty_node.nodeIbStasher.delay(ppgDelay(90)) # Long delay in Client Requests faulty_node.clientIbStasher.delay(req_delay(90)) return faulty_node, False
def test_forced_upgrade_handled_once_if_request_received_after_propagate( looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue): """ Verifies that POOL_UPGRADE force=true request is handled one time in case the node commits the transaction to the ledger but during the 3PC-process receives the request directly from the client after a PROPAGATE from some other node """ slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node slow_node.clientIbStasher.delay(req_delay()) slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta')) slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Gamma')) original_process_propagate = slow_node.nodeMsgRouter.routes[Propagate] original_process_request = slow_node.clientMsgRouter.routes[Request] def patched_process_propagate(msg: Propagate, frm: str): original_process_propagate(msg, frm) slow_node.clientIbStasher.reset_delays_and_process_delayeds() slow_node.nodeMsgRouter.routes[Propagate] = original_process_propagate def patched_process_request(request: Request, frm: str): original_process_request(request, frm) slow_node.nodeIbStasher.reset_delays_and_process_delayeds() slow_node.clientMsgRouter.routes[Request] = original_process_request slow_node.nodeMsgRouter.routes[Propagate] = patched_process_propagate slow_node.clientMsgRouter.routes[Request] = patched_process_request init_len = len(list(slow_node.upgrader._actionLog)) sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, validUpgradeExpForceTrue) looper.runFor(waits.expectedUpgradeScheduled()) checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION]) if init_len == 0: # first upgrade - should be only one scheduled assert len(list(slow_node.upgrader._actionLog)) == 1 else: # one upgrade were already scheduled. we should cancel it and schedule new one # so action log should be increased by 2 assert len(list(slow_node.upgrader._actionLog)) == init_len + 2 assert slow_node.upgrader._actionLog.lastEvent[1] == UpgradeLog.SCHEDULED
def setup(txnPoolNodeSet): faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node faulty_node.nodeIbStasher.delay(ppgDelay()) faulty_node.clientIbStasher.delay(req_delay()) return faulty_node